Master File

User Inputs

output.var = params$output.var 

transform.abs = FALSE
log.pred = params$log.pred
norm.pred = FALSE
algo.forward.caret = params$algo.forward.caret
algo.backward.caret = params$algo.backward.caret
algo.stepwise.caret = params$algo.stepwise.caret
algo.LASSO.caret = params$algo.LASSO.caret
algo.LARS.caret = params$algo.LARS.caret
message("Parameters used for training/prediction: ")

## Parameters used for training/prediction:

str(params)

## List of 7
##  $ output.var         : chr "y3"
##  $ log.pred           : logi TRUE
##  $ algo.forward.caret : logi TRUE
##  $ algo.backward.caret: logi TRUE
##  $ algo.stepwise.caret: logi TRUE
##  $ algo.LASSO.caret   : logi TRUE
##  $ algo.LARS.caret    : logi TRUE

# Setup Labels
output.var.tr = if (log.pred == TRUE)  paste0(output.var,'.log') else  output.var.tr = output.var

Loading Data

feat  = read.csv('../../Data/features_highprec.csv')
labels = read.csv('../../Data/labels.csv')
predictors = names(dplyr::select(feat,-JobName))
data.ori = inner_join(feat,labels,by='JobName')
#data.ori = inner_join(feat,select_at(labels,c('JobName',output.var)),by='JobName')

Data validation

cc  = complete.cases(data.ori)
data.notComplete = data.ori[! cc,]
data = data.ori[cc,] %>% select_at(c(predictors,output.var,'JobName'))
message('Original cases: ',nrow(data.ori))

## Original cases: 10000

message('Non-Complete cases: ',nrow(data.notComplete))

## Non-Complete cases: 3020

message('Complete cases: ',nrow(data))

## Complete cases: 6980

summary(dplyr::select_at(data,c('JobName',output.var)))

##       JobName           y3        
##  Job_00001:   1   Min.   : 95.91  
##  Job_00002:   1   1st Qu.:118.29  
##  Job_00003:   1   Median :124.03  
##  Job_00004:   1   Mean   :125.40  
##  Job_00007:   1   3rd Qu.:131.06  
##  Job_00008:   1   Max.   :193.73  
##  (Other)  :6974

Output Variable

The Output Variable y3 shows right skewness, so will proceed with a log transformation

Histogram

df=gather(select_at(data,output.var))
ggplot(df, aes(x=value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() 

  #stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  

QQPlot

ggplot(gather(select_at(data,output.var)), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Trasformation of Output Variable from y3 to y3.log

if(log.pred==TRUE) data[[output.var.tr]] = log(data[[output.var]],10) else
  data[[output.var.tr]] = data[[output.var]]
df=gather(select_at(data,c(output.var,output.var.tr)))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=2)

ggplot(gather(select_at(data,c(output.var,output.var.tr))), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Best Normalizator y3

Normalization of y3 using bestNormalize package. (suggested orderNorm) This is cool, but I think is too far for the objective of the project

t=bestNormalize::bestNormalize(data[[output.var]])

t

## Best Normalizing transformation with 6980 Observations
##  Estimated Normality Statistics (Pearson P / df, lower => more normal):
##  - No transform: 2.9701 
##  - Box-Cox: 1.4689 
##  - Log_b(x+a): 2.0304 
##  - sqrt(x+a): 2.4534 
##  - exp(x): 749.244 
##  - arcsinh(x): 2.0308 
##  - Yeo-Johnson: 1.1886 
##  - orderNorm: 1.1943 
## Estimation method: Out-of-sample via CV with 10 folds and 5 repeats
##  
## Based off these, bestNormalize chose:
## Standardized Yeo-Johnson Transformation with 6980 nonmissing obs.:
##  Estimated statistics:
##  - lambda = -1.998639 
##  - mean (before standardization) = 0.5003083 
##  - sd (before standardization) = 5.108542e-06

qqnorm(data[[output.var]])

qqnorm(predict(t))

orderNorm() is a rank-based procedure by which the values of a vector are mapped to their percentile, which is then mapped to the same percentile of the normal distribution. Without the presence of ties, this essentially guarantees that the transformation leads to a uniform distribution

Predictors

All predictors show a Fat-Tail situation, where the two tails are very tall, and a low distribution around the mean. The orderNorm transformation can help (see [Best Normalizator] section)

Interesting Predictors

Histograms

cols = c('x11','x18','stat98','x7','stat110')
df=gather(select_at(data,cols))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=3)

# ggplot(gather(select_at(data,cols)), aes(sample=value)) + 
#   stat_qq()+
#   facet_wrap(~key, scales = 'free',ncol=2)

lapply(select_at(data,cols),summary)

## $x11
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## 9.000e-08 9.494e-08 1.001e-07 1.001e-07 1.052e-07 1.100e-07 
## 
## $x18
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   1.500   3.147   4.769   4.772   6.418   7.999 
## 
## $stat98
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## -2.998619 -1.551882 -0.015993 -0.005946  1.528405  2.999499 
## 
## $x7
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   0.700   1.266   1.854   1.852   2.446   3.000 
## 
## $stat110
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## -2.999543 -1.496865 -0.002193 -0.004129  1.504273  2.999563

Scatter plot vs. output variable **y3.log

d = gather(dplyr::select_at(data,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light green',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=3)

## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

All Predictors

Histograms

All indicators have a strong indication of Fat-Tails

df=gather(select_at(data,predictors))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=4)

Correlations

With Output Variable

#chart.Correlation(select(data,-JobName),  pch=21)
t=as.data.frame(round(cor(dplyr::select(data,-one_of(output.var.tr,'JobName'))
                          ,select_at(data,output.var.tr)),4))  %>%
  rownames_to_column(var='variable') %>% filter(variable != !!output.var) %>% arrange(-y3.log)
#DT::datatable(t)
message("Top Positive")

## Top Positive

kable(head(arrange(t,desc(y3.log)),20))

variable	y3.log
x18	0.3120
x7	0.2091
stat98	0.1784
x9	0.1127
x17	0.0611
x16	0.0489
x10	0.0472
x21	0.0412
x11	0.0322
x8	0.0318
stat156	0.0287
stat23	0.0234
stat100	0.0206
stat144	0.0203
stat59	0.0202
stat60	0.0199
stat195	0.0199
stat141	0.0194
stat73	0.0192
stat197	0.0185

message("Top Negative")

## Top Negative

kable(head(arrange(t,y3.log),20))

variable	y3.log
stat110	-0.1594
x4	-0.0603
stat13	-0.0345
stat41	-0.0345
stat14	-0.0317
stat149	-0.0309
stat113	-0.0279
stat4	-0.0248
stat106	-0.0236
stat146	-0.0236
stat186	-0.0217
stat91	-0.0210
stat214	-0.0209
stat5	-0.0207
stat22	-0.0202
stat39	-0.0202
stat175	-0.0194
stat187	-0.0193
stat128	-0.0192
stat37	-0.0191

Between All Variables

#chart.Correlation(select(data,-JobName),  pch=21)
t=as.data.frame(round(cor(dplyr::select(data,-one_of('JobName'))),4))
#DT::datatable(t,options=list(scrollX=T))
message("Showing only 10 variables")

## Showing only 10 variables

kable(t[1:10,1:10])

	x1	x2	x3	x4	x5	x6	x7	x8	x9	x10
x1	1.0000	0.0034	-0.0028	0.0085	0.0068	0.0159	0.0264	-0.0012	0.0142	0.0013
x2	0.0034	1.0000	-0.0057	0.0004	-0.0094	-0.0101	0.0089	0.0078	0.0049	-0.0214
x3	-0.0028	-0.0057	1.0000	0.0029	0.0046	0.0006	-0.0105	-0.0002	0.0167	-0.0137
x4	0.0085	0.0004	0.0029	1.0000	-0.0059	0.0104	0.0098	0.0053	0.0061	-0.0023
x5	0.0068	-0.0094	0.0046	-0.0059	1.0000	0.0016	-0.0027	0.0081	0.0259	-0.0081
x6	0.0159	-0.0101	0.0006	0.0104	0.0016	1.0000	0.0200	-0.0157	0.0117	-0.0072
x7	0.0264	0.0089	-0.0105	0.0098	-0.0027	0.0200	1.0000	-0.0018	-0.0069	-0.0221
x8	-0.0012	0.0078	-0.0002	0.0053	0.0081	-0.0157	-0.0018	1.0000	0.0142	-0.0004
x9	0.0142	0.0049	0.0167	0.0061	0.0259	0.0117	-0.0069	0.0142	1.0000	0.0149
x10	0.0013	-0.0214	-0.0137	-0.0023	-0.0081	-0.0072	-0.0221	-0.0004	0.0149	1.0000

Scatter Plots with Output Variable

Scatter plots with all predictors and the output variable (y3.log)

d = gather(dplyr::select_at(data,c(predictors,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)

## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

Multicollinearity - VIF

No Multicollinearity among predictors

Showing Top predictor by VIF Value

vifDF = usdm::vif(select_at(data,predictors)) %>% arrange(desc(VIF))
head(vifDF,15)

##    Variables      VIF
## 1    stat207 1.062951
## 2    stat137 1.060574
## 3     stat31 1.060402
## 4    stat166 1.060395
## 5      stat6 1.059831
## 6    stat142 1.059533
## 7    stat121 1.059330
## 8    stat206 1.059192
## 9    stat202 1.059150
## 10   stat178 1.059092
## 11    stat20 1.058477
## 12   stat175 1.058138
## 13   stat112 1.057701
## 14   stat146 1.057548
## 15   stat156 1.057245

Feature Eng

• Square Root transformation for x18

data.tr=data %>%
  mutate(x18.sqrt = sqrt(x18)) 
cols=c('x18','x18.sqrt')

Comparing Pre and Post Transformation Density Plots

# ggplot(gather(select_at(data.tr,cols)), aes(value)) + 
#   geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
#   geom_density() + 
#   facet_wrap(~key, scales = 'free',ncol=4)

d = gather(dplyr::select_at(data.tr,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)

## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

#removing unwanted variables
data.tr=data.tr %>%
  dplyr::select_at(names(data.tr)[! names(data.tr) %in% c('x18','y3','JobName')])

data=data.tr
label.names=output.var.tr

Modeling

PCA

fullInteraction=1

pca.vars  = names(data)
pca.vars = pca.vars[!pca.vars %in% label.names]

if(fullInteraction){
  pca.formula =as.formula(paste0('~(',paste0(pca.vars, collapse ='+'),')^2'))
  pca.model = prcomp(formula=pca.formula,data=data[,pca.vars],center=T,scale.=T,retx = T)
  #saveRDS(pca.model,'pca.model.rds')
} else {
  pca.model =  prcomp(x=data[,pca.vars],center=T,scale.=T,retx = T)
}
targetCumVar = .8

pca.model$var = pca.model$sdev ^ 2 #eigenvalues
pca.model$pvar = pca.model$var / sum(pca.model$var)
pca.model$cumpvar = cumsum(pca.model$pvar )
pca.model$pcaSel = pca.model$cumpvar<=targetCumVar
pca.model$pcaSelCount = sum(pca.model$pcaSel)
pca.model$pcaSelTotVar = sum(pca.model$pvar[pca.model$pcaSel])
message(pca.model$pcaSelCount, " PCAs justify ",percent(targetCumVar)," of the total Variance. (",percent(pca.model$pcaSelTotVar),")")

## 3834 PCAs justify 80.0% of the total Variance. (80.0%)

plot(pca.model$var,xlab="Principal component", ylab="Proportion of variance explained",   type='b')

plot(cumsum(pca.model$pvar ),xlab="Principal component", ylab="Cumulative Proportion of variance explained", ylim=c(0,1), type='b')

screeplot(pca.model,npcs = pca.model$pcaSelCount)

screeplot(pca.model,npcs = pca.model$pcaSelCount,type='lines')

#summary(pca.model)
#pca.model$rotation
#creating dataset
data.pca = dplyr::select(data,!!label.names) %>% 
  dplyr::bind_cols(dplyr::select(as.data.frame(pca.model$x)
                                 ,!!colnames(pca.model$rotation)[pca.model$pcaSel])
  )

Train Test Split

data.pca = data[sample(nrow(data.pca)),] # randomly shuffle data
split = sample.split(data.pca[,label.names], SplitRatio = 0.8)

data.train = subset(data.pca, split == TRUE)
data.test = subset(data.pca, split == FALSE)

Common Functions

plot.diagnostics <-  function(model, train) {
  plot(model)
  
  residuals = resid(model) # Plotted above in plot(lm.out)
  r.standard = rstandard(model)
  r.student = rstudent(model)
  
  df = data.frame(x=predict(model,train),y=r.student)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = 0,size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  
  df = data.frame(x=predict(model,train),y=r.standard)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = c(-2,0,2),size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  # Histogram
  df=data.frame(r.student)
  p=ggplot(data=df,aes(r.student)) +
    geom_histogram(aes(y=..density..),bins = 50,fill='blue',alpha=0.6) + 
    stat_function(fun = dnorm, n = 100, args = list(mean = 0, sd = 1)) +
    ylab("Density")+
    xlab("Studentized Residuals")+
    ggtitle("Distribution of Studentized Residuals")
  plot(p)
  # http://www.stat.columbia.edu/~martin/W2024/R7.pdf
  # Influential plots
  inf.meas = influence.measures(model)
  # print (summary(inf.meas)) # too much data
  
  # Leverage plot
  lev = hat(model.matrix(model))
  df=tibble::rownames_to_column(as.data.frame(lev),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=lev)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    ylab('Leverage - check') + 
    xlab('Index')
  plot(p)
  # Cook's Distance
  cd = cooks.distance(model)
  df=tibble::rownames_to_column(as.data.frame(cd),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=cd)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_text(data=filter(df,cd>15/nrow(train)),aes(label=id),check_overlap=T,size=3,vjust=-.5)+
    ylab('Cooks distances') + 
    geom_hline(yintercept = c(4/nrow(train),0),size=1)+
    xlab('Index')
  plot(p)
  print (paste("Number of data points that have Cook's D > 4/n: ", length(cd[cd > 4/nrow(train)]), sep = "")) 
  print (paste("Number of data points that have Cook's D > 1: ", length(cd[cd > 1]), sep = "")) 
  return(cd)
}

# function to set up random seeds
# Based on http://jaehyeon-kim.github.io/2015/05/Setup-Random-Seeds-on-Caret-Package.html 
setCaretSeeds <- function(method = "cv", numbers = 1, repeats = 1, tunes = NULL, seed = 1701) {
  #B is the number of resamples and integer vector of M (numbers + tune length if any)
  B <- if (method == "cv") numbers
  else if(method == "repeatedcv") numbers * repeats
  else NULL
  if(is.null(length)) {
    seeds <- NULL
  } else {
    set.seed(seed = seed)
    seeds <- vector(mode = "list", length = B)
    seeds <- lapply(seeds, function(x) sample.int(n = 1000000
                                                  , size = numbers + ifelse(is.null(tunes), 0, tunes)))
    seeds[[length(seeds) + 1]] <- sample.int(n = 1000000, size = 1)
  }
  # return seeds
  seeds
}



train.caret.glmselect = function(formula, data, method
                                 ,subopt = NULL, feature.names
                                 , train.control = NULL, tune.grid = NULL, pre.proc = NULL){
  
  if(is.null(train.control)){
    train.control <- trainControl(method = "cv"
                              ,number = 10
                              ,seeds = setCaretSeeds(method = "cv"
                                                     , numbers = 10
                                                     , seed = 1701)
                              ,search = "grid"
                              ,verboseIter = TRUE
                              ,allowParallel = TRUE
                              )
  }
  
  if(is.null(tune.grid)){
    if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
      tune.grid = data.frame(nvmax = 1:length(feature.names))
    }
    if (method == 'glmnet' && subopt == 'LASSO'){
      # Will only show 1 Lambda value during training, but that is OK
      # https://stackoverflow.com/questions/47526544/why-need-to-tune-lambda-with-carettrain-method-glmnet-and-cv-glmnet
      # Another option for LASSO is this: https://github.com/topepo/caret/blob/master/RegressionTests/Code/lasso.R
      lambda = 10^seq(-2,0, length =100)
      alpha = c(1)
      tune.grid = expand.grid(alpha = alpha,lambda = lambda)
    }
    if (method == 'lars'){
      # https://github.com/topepo/caret/blob/master/RegressionTests/Code/lars.R
      fraction = seq(0, 1, length = 100)
      tune.grid = expand.grid(fraction = fraction)
      pre.proc = c("center", "scale") 
    }
  }
  
  # http://sshaikh.org/2015/05/06/parallelize-machine-learning-in-r-with-multi-core-cpus/
  cl <- makeCluster(ceiling(detectCores()*0.85)) # use 75% of cores only, leave rest for other tasks
  registerDoParallel(cl)

  set.seed(1) 
  # note that the seed has to actually be set just before this function is called
  # settign is above just not ensure reproducibility for some reason
  model.caret <- caret::train(formula
                              , data = data
                              , method = method
                              , tuneGrid = tune.grid
                              , trControl = train.control
                              , preProc = pre.proc
                              )
  
  stopCluster(cl)
  registerDoSEQ() # register sequential engine in case you are not using this function anymore
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    print("All models results")
    print(model.caret$results) # all model results
    print("Best Model")
    print(model.caret$bestTune) # best model
    model = model.caret$finalModel

    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-nvmax) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=nvmax,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    # leap function does not support studentized residuals
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)
   
    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    id = rownames(model.caret$bestTune)    
    # Provides the coefficients of the best model
    # regsubsets doens return a full model (see documentation of regsubset), so we need to recalcualte themodel
    # https://stackoverflow.com/questions/13063762/how-to-obtain-a-lm-object-from-regsubsets
    print("Coefficients of final model:")
    coefs <- coef(model, id=id)
    #calculate the model to the the coef intervals
    nams <- names(coefs)
    nams <- nams[!nams %in% "(Intercept)"]
    response <-  as.character(formula[[2]])
    form <- as.formula(paste(response, paste(nams, collapse = " + "), sep = " ~ "))
    mod <- lm(form, data = data)
    #coefs
    #coef(mod)
    print(car::Confint(mod))
    return(list(model = model,id = id, residPlot = residPlot, residHistogram=residHistogram
                ,modelLM=mod))
  }
  if (method == 'glmnet' && subopt == 'LASSO'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    print(model.caret$results)
    model=model.caret$finalModel
    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-lambda) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=lambda,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot 
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') +
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    #no interval for glmnet: https://stackoverflow.com/questions/39750965/confidence-intervals-for-ridge-regression
    t=coef(model,s=model.caret$bestTune$lambda)
    model.coef = t[which(t[,1]!=0),]
    print(as.data.frame(model.coef))
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, metricsPlot=metricsPlot ))
  }
  if (method == 'lars'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    # Metrics Plot
    dataPlot = model.caret$results %>%
        gather(key='metric',value='value',-fraction) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=fraction,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    t=coef(model.caret$finalModel,s=model.caret$bestTune$fraction,mode='fraction')
    model.coef = t[which(t!=0)]
    print(model.coef)
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, residHistogram=residHistogram))
  }
}

# https://stackoverflow.com/questions/48265743/linear-model-subset-selection-goodness-of-fit-with-k-fold-cross-validation
# changed slightly since call[[2]] was just returning "formula" without actually returnign the value in formula
predict.regsubsets <- function(object, newdata, id, formula, ...) {
    #form <- as.formula(object$call[[2]])
    mat <- model.matrix(formula, newdata) # adds intercept and expands any interaction terms
    coefi <- coef(object, id = id)
    xvars <- names(coefi)
    return(mat[,xvars]%*%coefi)
}
  
test.model = function(model, test, level=0.95
                      ,draw.limits = FALSE, good = 0.1, ok = 0.15
                      ,method = NULL, subopt = NULL
                      ,id = NULL, formula, feature.names, label.names
                      ,transformation = NULL){
  ## if using caret for glm select equivalent functionality, 
  ## need to pass formula (full is ok as it will select subset of variables from there)
  if (is.null(method)){
    pred = predict(model, newdata=test, interval="confidence", level = level) 
  }
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    pred = predict.regsubsets(model, newdata = test, id = id, formula = formula)
  }
  
  if (method == 'glmnet' && subopt == 'LASSO'){
    xtest = as.matrix(test[,feature.names]) 
    pred=as.data.frame(predict(model, xtest))
  }
  
  if (method == 'lars'){
    pred=as.data.frame(predict(model, newdata = test))
  }
    
  # Summary of predicted values
  print ("Summary of predicted values: ")
  print(summary(pred[,1]))

  test.mse = mean((test[,label.names]-pred[,1])^2)
  print (paste(method, subopt, "Test MSE:", test.mse, sep=" "))
  
  if(log.pred == TRUE || norm.pred == TRUE){
    # plot transformewd comparison first
    df=data.frame(x=test[,label.names],y=pred[,1])
    ggplot(df,aes(x=x,y=y)) +
      geom_point(color='blue',alpha=0.5,shape=20,size=2) +
      geom_abline(slope=1,intercept=0,color='black',size=1) +
      #scale_y_continuous(limits=c(min(df),max(df)))+
      xlab("Actual (Transformed)")+
      ylab("Predicted (Transformed)")
  }
    
  if (log.pred == FALSE && norm.pred == FALSE){
    x = test[,label.names]
    y = pred[,1]
  }
  if (log.pred == TRUE){
    x = 10^test[,label.names]
    y = 10^pred[,1]  
  }
  if (norm.pred == TRUE){
    x = predict(transformation, test[,label.names], inverse = TRUE)
    y = predict(transformation, pred[,1], inverse = TRUE)
  }

  df=data.frame(x,y)
  ggplot(df,aes(x,y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_abline(slope=c(1+good,1-good,1+ok,1-ok)
                ,intercept=rep(0,4),color=c('dark green','dark green','dark red','dark red'),size=1,alpha=0.8) +
    #scale_y_continuous(limits=c(min(df),max(df)))+
    xlab("Actual")+
    ylab("Predicted") 
    
 
}

Setup Formulae

n <- names(data.train)
 formula <- as.formula(paste(paste(n[n %in% label.names], collapse = " + ")
                             ," ~", paste(n[!n %in% label.names], collapse = " + "))) 

grand.mean.formula = as.formula(paste(paste(n[n %in% label.names], collapse = " + ")," ~ 1"))

print(formula)

## y3.log ~ x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + 
##     x12 + x13 + x14 + x15 + x16 + x17 + x19 + x20 + x21 + x22 + 
##     x23 + stat1 + stat2 + stat3 + stat4 + stat5 + stat6 + stat7 + 
##     stat8 + stat9 + stat10 + stat11 + stat12 + stat13 + stat14 + 
##     stat15 + stat16 + stat17 + stat18 + stat19 + stat20 + stat21 + 
##     stat22 + stat23 + stat24 + stat25 + stat26 + stat27 + stat28 + 
##     stat29 + stat30 + stat31 + stat32 + stat33 + stat34 + stat35 + 
##     stat36 + stat37 + stat38 + stat39 + stat40 + stat41 + stat42 + 
##     stat43 + stat44 + stat45 + stat46 + stat47 + stat48 + stat49 + 
##     stat50 + stat51 + stat52 + stat53 + stat54 + stat55 + stat56 + 
##     stat57 + stat58 + stat59 + stat60 + stat61 + stat62 + stat63 + 
##     stat64 + stat65 + stat66 + stat67 + stat68 + stat69 + stat70 + 
##     stat71 + stat72 + stat73 + stat74 + stat75 + stat76 + stat77 + 
##     stat78 + stat79 + stat80 + stat81 + stat82 + stat83 + stat84 + 
##     stat85 + stat86 + stat87 + stat88 + stat89 + stat90 + stat91 + 
##     stat92 + stat93 + stat94 + stat95 + stat96 + stat97 + stat98 + 
##     stat99 + stat100 + stat101 + stat102 + stat103 + stat104 + 
##     stat105 + stat106 + stat107 + stat108 + stat109 + stat110 + 
##     stat111 + stat112 + stat113 + stat114 + stat115 + stat116 + 
##     stat117 + stat118 + stat119 + stat120 + stat121 + stat122 + 
##     stat123 + stat124 + stat125 + stat126 + stat127 + stat128 + 
##     stat129 + stat130 + stat131 + stat132 + stat133 + stat134 + 
##     stat135 + stat136 + stat137 + stat138 + stat139 + stat140 + 
##     stat141 + stat142 + stat143 + stat144 + stat145 + stat146 + 
##     stat147 + stat148 + stat149 + stat150 + stat151 + stat152 + 
##     stat153 + stat154 + stat155 + stat156 + stat157 + stat158 + 
##     stat159 + stat160 + stat161 + stat162 + stat163 + stat164 + 
##     stat165 + stat166 + stat167 + stat168 + stat169 + stat170 + 
##     stat171 + stat172 + stat173 + stat174 + stat175 + stat176 + 
##     stat177 + stat178 + stat179 + stat180 + stat181 + stat182 + 
##     stat183 + stat184 + stat185 + stat186 + stat187 + stat188 + 
##     stat189 + stat190 + stat191 + stat192 + stat193 + stat194 + 
##     stat195 + stat196 + stat197 + stat198 + stat199 + stat200 + 
##     stat201 + stat202 + stat203 + stat204 + stat205 + stat206 + 
##     stat207 + stat208 + stat209 + stat210 + stat211 + stat212 + 
##     stat213 + stat214 + stat215 + stat216 + stat217 + x18.sqrt

print(grand.mean.formula)

## y3.log ~ 1

# Update feature.names because we may have transformed some features
feature.names = n[!n %in% label.names]

Full Model

model.full = lm(formula , data.train)
summary(model.full)

## 
## Call:
## lm(formula = formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.087823 -0.020936 -0.004818  0.016147  0.188961 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.975e+00  9.630e-03 205.059  < 2e-16 ***
## x1          -2.387e-04  6.596e-04  -0.362 0.717442    
## x2           1.606e-04  4.231e-04   0.380 0.704263    
## x3          -1.249e-05  1.156e-04  -0.108 0.913983    
## x4          -4.397e-05  9.085e-06  -4.841 1.33e-06 ***
## x5           5.205e-04  2.971e-04   1.752 0.079877 .  
## x6           4.011e-04  6.033e-04   0.665 0.506146    
## x7           1.135e-02  6.462e-04  17.560  < 2e-16 ***
## x8           4.261e-04  1.498e-04   2.844 0.004470 ** 
## x9           3.307e-03  3.346e-04   9.886  < 2e-16 ***
## x10          1.198e-03  3.115e-04   3.846 0.000121 ***
## x11          1.625e+05  7.442e+04   2.184 0.028998 *  
## x12         -1.228e-04  1.901e-04  -0.646 0.518362    
## x13          6.745e-05  7.560e-05   0.892 0.372311    
## x14         -3.510e-04  3.264e-04  -1.075 0.282313    
## x15          9.911e-05  3.096e-04   0.320 0.748911    
## x16          6.822e-04  2.160e-04   3.159 0.001592 ** 
## x17          1.525e-03  3.280e-04   4.651 3.38e-06 ***
## x19          2.325e-04  1.678e-04   1.385 0.166009    
## x20         -2.204e-04  1.160e-03  -0.190 0.849337    
## x21          1.265e-04  4.276e-05   2.959 0.003100 ** 
## x22         -3.904e-04  3.473e-04  -1.124 0.261074    
## x23         -1.925e-04  3.305e-04  -0.583 0.560216    
## stat1       -2.356e-04  2.493e-04  -0.945 0.344825    
## stat2        8.538e-05  2.494e-04   0.342 0.732048    
## stat3        3.428e-04  2.503e-04   1.370 0.170841    
## stat4       -3.667e-04  2.534e-04  -1.447 0.148028    
## stat5        6.690e-06  2.526e-04   0.026 0.978876    
## stat6       -1.419e-04  2.506e-04  -0.566 0.571200    
## stat7       -6.248e-05  2.522e-04  -0.248 0.804295    
## stat8        2.401e-04  2.502e-04   0.960 0.337282    
## stat9        7.159e-05  2.508e-04   0.285 0.775333    
## stat10      -2.328e-04  2.513e-04  -0.926 0.354271    
## stat11      -1.457e-04  2.532e-04  -0.575 0.565003    
## stat12       3.127e-04  2.504e-04   1.248 0.211914    
## stat13      -5.756e-04  2.494e-04  -2.308 0.021032 *  
## stat14      -8.059e-04  2.486e-04  -3.241 0.001197 ** 
## stat15      -3.954e-04  2.492e-04  -1.587 0.112575    
## stat16       7.597e-05  2.503e-04   0.304 0.761505    
## stat17      -1.442e-04  2.490e-04  -0.579 0.562526    
## stat18      -2.127e-04  2.498e-04  -0.852 0.394450    
## stat19       1.500e-04  2.496e-04   0.601 0.547867    
## stat20      -9.038e-05  2.501e-04  -0.361 0.717867    
## stat21      -2.529e-05  2.517e-04  -0.100 0.919987    
## stat22      -6.685e-04  2.505e-04  -2.668 0.007649 ** 
## stat23       7.748e-04  2.502e-04   3.097 0.001967 ** 
## stat24      -7.147e-04  2.512e-04  -2.846 0.004450 ** 
## stat25      -4.518e-04  2.493e-04  -1.812 0.069987 .  
## stat26      -3.711e-04  2.501e-04  -1.484 0.137993    
## stat27       2.482e-04  2.508e-04   0.990 0.322415    
## stat28      -6.528e-06  2.505e-04  -0.026 0.979213    
## stat29       2.612e-04  2.528e-04   1.033 0.301477    
## stat30       3.596e-04  2.526e-04   1.424 0.154514    
## stat31      -4.958e-05  2.531e-04  -0.196 0.844666    
## stat32       3.993e-04  2.523e-04   1.583 0.113468    
## stat33      -1.760e-04  2.497e-04  -0.705 0.480846    
## stat34      -4.333e-05  2.498e-04  -0.173 0.862298    
## stat35      -3.934e-04  2.514e-04  -1.565 0.117753    
## stat36      -3.094e-05  2.481e-04  -0.125 0.900793    
## stat37      -4.522e-04  2.526e-04  -1.790 0.073433 .  
## stat38       4.082e-04  2.513e-04   1.624 0.104389    
## stat39      -3.480e-04  2.494e-04  -1.395 0.163012    
## stat40      -2.057e-05  2.511e-04  -0.082 0.934696    
## stat41      -3.916e-04  2.492e-04  -1.572 0.116104    
## stat42      -7.757e-05  2.499e-04  -0.310 0.756286    
## stat43      -3.372e-04  2.507e-04  -1.345 0.178706    
## stat44       5.649e-05  2.525e-04   0.224 0.823011    
## stat45      -3.576e-04  2.488e-04  -1.437 0.150661    
## stat46       3.615e-04  2.506e-04   1.442 0.149294    
## stat47       2.193e-04  2.521e-04   0.870 0.384430    
## stat48       1.328e-04  2.511e-04   0.529 0.596912    
## stat49      -1.785e-05  2.485e-04  -0.072 0.942753    
## stat50       4.444e-04  2.485e-04   1.788 0.073806 .  
## stat51       4.969e-04  2.510e-04   1.980 0.047748 *  
## stat52      -2.364e-05  2.516e-04  -0.094 0.925118    
## stat53      -2.170e-04  2.526e-04  -0.859 0.390389    
## stat54      -4.232e-04  2.515e-04  -1.682 0.092545 .  
## stat55       3.422e-04  2.488e-04   1.376 0.169030    
## stat56      -4.455e-05  2.523e-04  -0.177 0.859827    
## stat57      -5.070e-05  2.473e-04  -0.205 0.837594    
## stat58       1.508e-04  2.493e-04   0.605 0.545353    
## stat59       3.180e-04  2.521e-04   1.261 0.207234    
## stat60       4.982e-04  2.512e-04   1.983 0.047400 *  
## stat61      -1.615e-05  2.520e-04  -0.064 0.948899    
## stat62       1.236e-04  2.498e-04   0.495 0.620865    
## stat63       4.348e-04  2.525e-04   1.722 0.085120 .  
## stat64      -1.120e-04  2.505e-04  -0.447 0.654842    
## stat65      -4.327e-04  2.516e-04  -1.720 0.085482 .  
## stat66       1.192e-04  2.541e-04   0.469 0.638963    
## stat67       1.168e-04  2.518e-04   0.464 0.642762    
## stat68      -7.627e-05  2.514e-04  -0.303 0.761590    
## stat69      -1.993e-04  2.498e-04  -0.798 0.425051    
## stat70       2.298e-04  2.494e-04   0.921 0.356952    
## stat71       1.194e-04  2.500e-04   0.478 0.632913    
## stat72       1.345e-04  2.513e-04   0.535 0.592373    
## stat73       3.373e-04  2.507e-04   1.346 0.178501    
## stat74      -2.392e-04  2.533e-04  -0.944 0.345169    
## stat75      -3.282e-04  2.531e-04  -1.297 0.194725    
## stat76       2.726e-04  2.511e-04   1.085 0.277837    
## stat77       7.751e-05  2.506e-04   0.309 0.757155    
## stat78      -1.470e-04  2.510e-04  -0.586 0.558200    
## stat79      -1.147e-04  2.516e-04  -0.456 0.648475    
## stat80       2.112e-04  2.514e-04   0.840 0.400866    
## stat81       2.846e-04  2.509e-04   1.134 0.256643    
## stat82       1.995e-04  2.493e-04   0.800 0.423649    
## stat83      -1.159e-04  2.501e-04  -0.463 0.643117    
## stat84       2.899e-05  2.504e-04   0.116 0.907838    
## stat85       2.604e-04  2.517e-04   1.034 0.300974    
## stat86       2.713e-04  2.502e-04   1.084 0.278446    
## stat87      -4.699e-04  2.511e-04  -1.872 0.061301 .  
## stat88      -6.743e-05  2.491e-04  -0.271 0.786634    
## stat89      -1.364e-04  2.478e-04  -0.550 0.582116    
## stat90      -8.980e-05  2.509e-04  -0.358 0.720412    
## stat91      -3.304e-04  2.487e-04  -1.328 0.184126    
## stat92      -4.869e-04  2.521e-04  -1.932 0.053448 .  
## stat93       9.592e-06  2.533e-04   0.038 0.969795    
## stat94      -3.932e-04  2.520e-04  -1.560 0.118754    
## stat95      -4.614e-04  2.510e-04  -1.838 0.066051 .  
## stat96      -2.965e-04  2.494e-04  -1.189 0.234609    
## stat97       2.322e-04  2.477e-04   0.937 0.348573    
## stat98       3.502e-03  2.466e-04  14.204  < 2e-16 ***
## stat99       3.843e-04  2.523e-04   1.524 0.127673    
## stat100      6.738e-04  2.501e-04   2.694 0.007087 ** 
## stat101     -1.245e-04  2.536e-04  -0.491 0.623634    
## stat102     -2.460e-04  2.530e-04  -0.973 0.330805    
## stat103     -2.711e-04  2.537e-04  -1.069 0.285200    
## stat104     -3.447e-04  2.508e-04  -1.374 0.169358    
## stat105      7.012e-05  2.495e-04   0.281 0.778671    
## stat106     -4.950e-04  2.501e-04  -1.979 0.047864 *  
## stat107     -4.943e-05  2.489e-04  -0.199 0.842586    
## stat108     -9.507e-05  2.508e-04  -0.379 0.704657    
## stat109     -6.885e-05  2.498e-04  -0.276 0.782852    
## stat110     -3.408e-03  2.505e-04 -13.605  < 2e-16 ***
## stat111     -1.479e-04  2.495e-04  -0.593 0.553439    
## stat112     -3.040e-04  2.522e-04  -1.205 0.228094    
## stat113     -2.801e-04  2.520e-04  -1.111 0.266497    
## stat114     -6.692e-05  2.491e-04  -0.269 0.788185    
## stat115      3.161e-04  2.496e-04   1.266 0.205510    
## stat116      2.466e-04  2.517e-04   0.980 0.327154    
## stat117      2.054e-04  2.505e-04   0.820 0.412221    
## stat118     -2.001e-04  2.478e-04  -0.807 0.419581    
## stat119      9.629e-05  2.486e-04   0.387 0.698517    
## stat120      8.546e-05  2.492e-04   0.343 0.731690    
## stat121     -1.911e-04  2.510e-04  -0.761 0.446484    
## stat122     -5.084e-05  2.484e-04  -0.205 0.837826    
## stat123      2.107e-04  2.540e-04   0.830 0.406858    
## stat124     -1.455e-04  2.515e-04  -0.579 0.562766    
## stat125      4.498e-05  2.519e-04   0.179 0.858296    
## stat126      1.123e-04  2.507e-04   0.448 0.654156    
## stat127      1.111e-04  2.498e-04   0.445 0.656350    
## stat128     -6.007e-05  2.493e-04  -0.241 0.809644    
## stat129      1.943e-04  2.493e-04   0.779 0.435840    
## stat130      3.403e-04  2.522e-04   1.349 0.177260    
## stat131      1.753e-04  2.517e-04   0.696 0.486304    
## stat132     -7.555e-06  2.496e-04  -0.030 0.975856    
## stat133      2.194e-04  2.508e-04   0.875 0.381821    
## stat134     -2.577e-04  2.489e-04  -1.035 0.300513    
## stat135      9.441e-05  2.505e-04   0.377 0.706302    
## stat136     -7.168e-05  2.523e-04  -0.284 0.776362    
## stat137      1.010e-04  2.482e-04   0.407 0.684121    
## stat138      8.468e-05  2.505e-04   0.338 0.735306    
## stat139      3.312e-05  2.512e-04   0.132 0.895129    
## stat140     -2.539e-04  2.490e-04  -1.020 0.307895    
## stat141      2.413e-04  2.499e-04   0.966 0.334211    
## stat142     -1.488e-04  2.533e-04  -0.588 0.556832    
## stat143      2.042e-04  2.500e-04   0.817 0.414195    
## stat144      5.299e-04  2.496e-04   2.123 0.033818 *  
## stat145     -1.074e-04  2.538e-04  -0.423 0.672076    
## stat146     -4.488e-04  2.521e-04  -1.780 0.075093 .  
## stat147     -4.115e-04  2.518e-04  -1.634 0.102295    
## stat148     -2.412e-04  2.487e-04  -0.970 0.332207    
## stat149     -5.992e-04  2.519e-04  -2.379 0.017388 *  
## stat150     -8.703e-05  2.522e-04  -0.345 0.730063    
## stat151     -1.633e-04  2.526e-04  -0.646 0.518125    
## stat152     -3.056e-04  2.488e-04  -1.229 0.219303    
## stat153      5.868e-06  2.538e-04   0.023 0.981557    
## stat154     -9.471e-05  2.534e-04  -0.374 0.708583    
## stat155     -1.421e-04  2.481e-04  -0.573 0.566926    
## stat156      4.839e-04  2.535e-04   1.909 0.056335 .  
## stat157      2.549e-05  2.488e-04   0.102 0.918407    
## stat158      5.433e-06  2.534e-04   0.021 0.982894    
## stat159      3.627e-04  2.507e-04   1.447 0.147940    
## stat160     -1.260e-04  2.514e-04  -0.501 0.616222    
## stat161      2.147e-04  2.511e-04   0.855 0.392577    
## stat162      1.679e-05  2.495e-04   0.067 0.946350    
## stat163      6.746e-05  2.531e-04   0.266 0.789879    
## stat164      9.088e-05  2.541e-04   0.358 0.720555    
## stat165      6.278e-05  2.489e-04   0.252 0.800857    
## stat166     -2.529e-04  2.482e-04  -1.019 0.308170    
## stat167     -2.202e-04  2.493e-04  -0.883 0.377174    
## stat168     -2.163e-04  2.499e-04  -0.866 0.386776    
## stat169      5.172e-05  2.508e-04   0.206 0.836619    
## stat170     -4.443e-04  2.512e-04  -1.768 0.077039 .  
## stat171     -3.331e-05  2.520e-04  -0.132 0.894862    
## stat172      4.010e-04  2.490e-04   1.610 0.107353    
## stat173     -3.757e-04  2.516e-04  -1.494 0.135363    
## stat174      1.268e-05  2.506e-04   0.051 0.959643    
## stat175     -4.957e-04  2.507e-04  -1.978 0.048026 *  
## stat176      2.491e-04  2.511e-04   0.992 0.321190    
## stat177     -3.028e-05  2.525e-04  -0.120 0.904553    
## stat178     -1.761e-04  2.548e-04  -0.691 0.489447    
## stat179     -6.063e-06  2.485e-04  -0.024 0.980539    
## stat180     -2.505e-04  2.486e-04  -1.008 0.313618    
## stat181      2.090e-04  2.529e-04   0.826 0.408638    
## stat182      2.363e-04  2.523e-04   0.936 0.349083    
## stat183      1.453e-04  2.509e-04   0.579 0.562570    
## stat184     -1.816e-05  2.544e-04  -0.071 0.943087    
## stat185     -4.392e-04  2.478e-04  -1.772 0.076415 .  
## stat186     -1.832e-04  2.526e-04  -0.725 0.468297    
## stat187     -5.102e-04  2.505e-04  -2.037 0.041720 *  
## stat188     -1.755e-04  2.499e-04  -0.702 0.482592    
## stat189     -4.887e-05  2.511e-04  -0.195 0.845695    
## stat190      1.163e-04  2.491e-04   0.467 0.640759    
## stat191     -4.392e-04  2.504e-04  -1.754 0.079506 .  
## stat192      2.150e-04  2.532e-04   0.849 0.395970    
## stat193      1.595e-04  2.529e-04   0.630 0.528423    
## stat194      1.189e-04  2.493e-04   0.477 0.633499    
## stat195      2.354e-04  2.503e-04   0.941 0.346919    
## stat196     -2.817e-04  2.539e-04  -1.110 0.267257    
## stat197      2.144e-05  2.480e-04   0.086 0.931118    
## stat198     -2.001e-04  2.515e-04  -0.796 0.426248    
## stat199      3.489e-04  2.494e-04   1.399 0.161937    
## stat200     -2.205e-04  2.476e-04  -0.890 0.373276    
## stat201      5.049e-05  2.505e-04   0.202 0.840235    
## stat202     -2.250e-04  2.543e-04  -0.885 0.376329    
## stat203      6.231e-05  2.499e-04   0.249 0.803157    
## stat204     -4.822e-04  2.483e-04  -1.942 0.052136 .  
## stat205     -6.006e-05  2.492e-04  -0.241 0.809563    
## stat206     -1.047e-04  2.512e-04  -0.417 0.677022    
## stat207      3.507e-04  2.500e-04   1.403 0.160710    
## stat208     -9.681e-05  2.511e-04  -0.386 0.699863    
## stat209     -2.701e-04  2.490e-04  -1.085 0.278128    
## stat210     -2.866e-04  2.503e-04  -1.145 0.252266    
## stat211     -1.123e-04  2.503e-04  -0.449 0.653704    
## stat212      8.265e-05  2.504e-04   0.330 0.741371    
## stat213     -1.577e-04  2.525e-04  -0.624 0.532336    
## stat214     -3.493e-04  2.512e-04  -1.391 0.164384    
## stat215     -3.036e-05  2.502e-04  -0.121 0.903426    
## stat216     -1.871e-04  2.510e-04  -0.746 0.455968    
## stat217      3.134e-04  2.504e-04   1.251 0.210830    
## x18.sqrt     2.533e-02  9.591e-04  26.407  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03173 on 5343 degrees of freedom
## Multiple R-squared:  0.2633, Adjusted R-squared:  0.2302 
## F-statistic: 7.958 on 240 and 5343 DF,  p-value: < 2.2e-16

cd.full = plot.diagnostics(model=model.full, train=data.train)

## [1] "Number of data points that have Cook's D > 4/n: 288"
## [1] "Number of data points that have Cook's D > 1: 0"

Checking with removal of high influence points

high.cd = names(cd.full[cd.full > 4/nrow(data.train)])

#save dataset with high.cd flagged
t = data.train %>% 
  rownames_to_column() %>%
  mutate(high.cd = ifelse(rowname %in% high.cd,1,0))
#write.csv(t,file='data_high_cd_flag.csv',row.names = F)
###
data.train2 = data.train[!(rownames(data.train)) %in% high.cd,]
model.full2 = lm(formula , data.train2)
summary(model.full2)

## 
## Call:
## lm(formula = formula, data = data.train2)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.060310 -0.017361 -0.002699  0.016784  0.069654 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.954e+00  7.807e-03 250.340  < 2e-16 ***
## x1          -3.136e-04  5.384e-04  -0.582 0.560265    
## x2           3.169e-04  3.434e-04   0.923 0.356109    
## x3           5.197e-05  9.348e-05   0.556 0.578284    
## x4          -5.738e-05  7.396e-06  -7.758 1.03e-14 ***
## x5           5.671e-04  2.427e-04   2.336 0.019514 *  
## x6           1.564e-04  4.906e-04   0.319 0.749869    
## x7           1.132e-02  5.253e-04  21.554  < 2e-16 ***
## x8           4.704e-04  1.215e-04   3.872 0.000109 ***
## x9           3.105e-03  2.704e-04  11.482  < 2e-16 ***
## x10          1.619e-03  2.528e-04   6.404 1.65e-10 ***
## x11          2.355e+05  6.041e+04   3.899 9.80e-05 ***
## x12          6.099e-05  1.535e-04   0.397 0.691244    
## x13          1.454e-04  6.195e-05   2.348 0.018924 *  
## x14         -3.679e-04  2.660e-04  -1.383 0.166636    
## x15         -1.277e-04  2.528e-04  -0.505 0.613624    
## x16          1.184e-03  1.754e-04   6.751 1.63e-11 ***
## x17          1.492e-03  2.686e-04   5.557 2.89e-08 ***
## x19          2.109e-04  1.360e-04   1.551 0.121080    
## x20         -2.583e-04  9.469e-04  -0.273 0.785059    
## x21          1.274e-04  3.460e-05   3.683 0.000233 ***
## x22         -6.687e-04  2.827e-04  -2.365 0.018074 *  
## x23         -5.181e-05  2.700e-04  -0.192 0.847843    
## stat1       -3.483e-04  2.042e-04  -1.706 0.088074 .  
## stat2        3.549e-04  2.023e-04   1.755 0.079405 .  
## stat3        2.952e-04  2.062e-04   1.432 0.152263    
## stat4       -6.610e-04  2.053e-04  -3.219 0.001293 ** 
## stat5       -1.258e-04  2.054e-04  -0.612 0.540298    
## stat6        1.951e-05  2.051e-04   0.095 0.924250    
## stat7       -1.804e-04  2.040e-04  -0.884 0.376721    
## stat8       -2.849e-05  2.029e-04  -0.140 0.888325    
## stat9       -8.125e-05  2.039e-04  -0.398 0.690297    
## stat10      -1.283e-04  2.039e-04  -0.629 0.529236    
## stat11      -5.274e-04  2.054e-04  -2.568 0.010267 *  
## stat12       2.050e-04  2.037e-04   1.006 0.314297    
## stat13      -3.620e-04  2.023e-04  -1.789 0.073616 .  
## stat14      -1.084e-03  2.024e-04  -5.357 8.85e-08 ***
## stat15      -6.300e-04  2.020e-04  -3.119 0.001822 ** 
## stat16      -2.151e-04  2.038e-04  -1.055 0.291296    
## stat17      -7.841e-05  2.026e-04  -0.387 0.698680    
## stat18       2.324e-05  2.032e-04   0.114 0.908981    
## stat19      -3.200e-05  2.043e-04  -0.157 0.875530    
## stat20      -1.811e-05  2.031e-04  -0.089 0.928968    
## stat21      -1.500e-05  2.043e-04  -0.073 0.941448    
## stat22      -4.380e-04  2.043e-04  -2.144 0.032088 *  
## stat23       4.724e-04  2.035e-04   2.322 0.020290 *  
## stat24      -5.808e-04  2.044e-04  -2.842 0.004502 ** 
## stat25      -3.175e-04  2.039e-04  -1.557 0.119534    
## stat26      -1.238e-04  2.032e-04  -0.610 0.542170    
## stat27      -1.701e-05  2.045e-04  -0.083 0.933728    
## stat28      -1.798e-04  2.035e-04  -0.883 0.377049    
## stat29       8.800e-05  2.048e-04   0.430 0.667429    
## stat30       8.695e-05  2.049e-04   0.424 0.671263    
## stat31      -7.657e-06  2.048e-04  -0.037 0.970173    
## stat32      -2.766e-05  2.056e-04  -0.135 0.892988    
## stat33      -5.570e-05  2.024e-04  -0.275 0.783148    
## stat34       4.537e-04  2.028e-04   2.237 0.025343 *  
## stat35      -4.589e-04  2.051e-04  -2.238 0.025286 *  
## stat36       2.120e-05  2.033e-04   0.104 0.916966    
## stat37       3.756e-05  2.044e-04   0.184 0.854218    
## stat38       5.286e-04  2.032e-04   2.602 0.009292 ** 
## stat39      -2.572e-04  2.032e-04  -1.266 0.205653    
## stat40       2.407e-04  2.047e-04   1.176 0.239657    
## stat41      -4.155e-04  2.024e-04  -2.053 0.040165 *  
## stat42      -1.552e-05  2.035e-04  -0.076 0.939210    
## stat43      -2.262e-05  2.043e-04  -0.111 0.911850    
## stat44       9.389e-05  2.027e-04   0.463 0.643257    
## stat45      -4.274e-05  2.023e-04  -0.211 0.832667    
## stat46       2.326e-06  2.041e-04   0.011 0.990910    
## stat47       3.359e-04  2.045e-04   1.643 0.100482    
## stat48       1.727e-04  2.041e-04   0.846 0.397678    
## stat49      -1.611e-04  2.027e-04  -0.795 0.426603    
## stat50       2.343e-04  2.019e-04   1.160 0.245954    
## stat51       4.254e-05  2.050e-04   0.207 0.835629    
## stat52       9.963e-05  2.046e-04   0.487 0.626386    
## stat53      -2.133e-04  2.064e-04  -1.034 0.301256    
## stat54      -3.428e-04  2.051e-04  -1.671 0.094687 .  
## stat55       2.826e-04  2.014e-04   1.403 0.160705    
## stat56       6.002e-05  2.046e-04   0.293 0.769318    
## stat57       1.627e-04  2.017e-04   0.807 0.419743    
## stat58      -2.826e-05  2.015e-04  -0.140 0.888500    
## stat59       2.567e-04  2.037e-04   1.260 0.207692    
## stat60       3.690e-04  2.034e-04   1.814 0.069711 .  
## stat61       1.570e-04  2.048e-04   0.766 0.443492    
## stat62      -2.185e-04  2.036e-04  -1.073 0.283221    
## stat63       1.596e-04  2.032e-04   0.785 0.432356    
## stat64       1.773e-04  2.023e-04   0.876 0.380918    
## stat65      -7.033e-05  2.041e-04  -0.345 0.730397    
## stat66       1.646e-04  2.058e-04   0.800 0.423645    
## stat67       1.985e-04  2.054e-04   0.966 0.334036    
## stat68      -8.308e-05  2.039e-04  -0.407 0.683727    
## stat69      -6.792e-05  2.042e-04  -0.333 0.739494    
## stat70       2.981e-04  2.029e-04   1.469 0.141890    
## stat71       4.023e-04  2.031e-04   1.981 0.047650 *  
## stat72       5.627e-05  2.047e-04   0.275 0.783351    
## stat73       5.067e-04  2.059e-04   2.461 0.013874 *  
## stat74       3.584e-05  2.035e-04   0.176 0.860174    
## stat75       1.745e-04  2.051e-04   0.851 0.394876    
## stat76       1.582e-04  2.046e-04   0.773 0.439295    
## stat77       5.558e-05  2.037e-04   0.273 0.784963    
## stat78      -2.849e-04  2.031e-04  -1.403 0.160760    
## stat79      -1.309e-04  2.036e-04  -0.643 0.520315    
## stat80       2.775e-04  2.051e-04   1.353 0.176180    
## stat81       2.787e-04  2.041e-04   1.365 0.172278    
## stat82       2.071e-04  2.035e-04   1.018 0.308802    
## stat83      -1.752e-05  2.040e-04  -0.086 0.931567    
## stat84      -7.968e-05  2.038e-04  -0.391 0.695827    
## stat85      -5.549e-05  2.030e-04  -0.273 0.784561    
## stat86       3.300e-04  2.044e-04   1.615 0.106424    
## stat87      -4.055e-04  2.047e-04  -1.981 0.047646 *  
## stat88       7.719e-05  2.013e-04   0.383 0.701386    
## stat89       2.055e-06  2.035e-04   0.010 0.991942    
## stat90      -2.298e-04  2.051e-04  -1.121 0.262525    
## stat91      -4.032e-04  2.031e-04  -1.985 0.047157 *  
## stat92      -2.096e-04  2.040e-04  -1.028 0.304189    
## stat93      -9.446e-05  2.074e-04  -0.455 0.648821    
## stat94       7.290e-05  2.028e-04   0.359 0.719294    
## stat95       1.153e-04  2.028e-04   0.568 0.569807    
## stat96      -2.239e-04  2.037e-04  -1.099 0.271678    
## stat97       1.540e-04  2.024e-04   0.761 0.446711    
## stat98       3.444e-03  2.008e-04  17.151  < 2e-16 ***
## stat99       3.586e-04  2.048e-04   1.751 0.080033 .  
## stat100      3.331e-04  2.045e-04   1.629 0.103403    
## stat101     -1.527e-05  2.048e-04  -0.075 0.940558    
## stat102      1.059e-04  2.043e-04   0.519 0.604100    
## stat103     -2.891e-04  2.059e-04  -1.404 0.160366    
## stat104     -2.192e-04  2.042e-04  -1.074 0.283096    
## stat105      5.882e-05  2.018e-04   0.292 0.770674    
## stat106     -3.807e-04  2.029e-04  -1.876 0.060668 .  
## stat107      1.804e-05  2.021e-04   0.089 0.928873    
## stat108     -3.151e-05  2.045e-04  -0.154 0.877548    
## stat109     -1.450e-04  2.026e-04  -0.715 0.474367    
## stat110     -3.135e-03  2.030e-04 -15.439  < 2e-16 ***
## stat111     -2.426e-04  2.030e-04  -1.195 0.232205    
## stat112     -2.078e-04  2.045e-04  -1.016 0.309442    
## stat113      8.077e-06  2.041e-04   0.040 0.968436    
## stat114      1.651e-04  2.039e-04   0.810 0.418068    
## stat115      4.326e-04  2.024e-04   2.137 0.032625 *  
## stat116      2.919e-04  2.033e-04   1.436 0.151069    
## stat117      1.205e-04  2.049e-04   0.588 0.556455    
## stat118      9.342e-06  2.017e-04   0.046 0.963058    
## stat119      1.084e-04  2.029e-04   0.534 0.593277    
## stat120      4.662e-05  2.018e-04   0.231 0.817298    
## stat121     -1.812e-04  2.038e-04  -0.889 0.374009    
## stat122     -2.987e-04  2.025e-04  -1.475 0.140325    
## stat123      1.359e-05  2.056e-04   0.066 0.947285    
## stat124     -2.422e-04  2.040e-04  -1.187 0.235206    
## stat125      1.931e-04  2.037e-04   0.948 0.343184    
## stat126      1.178e-04  2.032e-04   0.580 0.562149    
## stat127      1.050e-04  2.033e-04   0.517 0.605391    
## stat128      5.778e-06  2.024e-04   0.029 0.977224    
## stat129      3.888e-05  2.023e-04   0.192 0.847639    
## stat130      8.411e-05  2.030e-04   0.414 0.678591    
## stat131      8.275e-05  2.043e-04   0.405 0.685529    
## stat132     -1.710e-04  2.011e-04  -0.850 0.395195    
## stat133      4.090e-04  2.039e-04   2.006 0.044901 *  
## stat134     -1.846e-04  2.037e-04  -0.906 0.364778    
## stat135     -2.380e-04  2.043e-04  -1.165 0.244197    
## stat136     -1.823e-04  2.047e-04  -0.890 0.373282    
## stat137      1.160e-04  2.010e-04   0.577 0.563809    
## stat138     -6.555e-05  2.040e-04  -0.321 0.747925    
## stat139     -7.649e-06  2.045e-04  -0.037 0.970162    
## stat140      1.587e-04  2.018e-04   0.787 0.431578    
## stat141      2.354e-04  2.029e-04   1.160 0.246021    
## stat142      5.978e-05  2.049e-04   0.292 0.770473    
## stat143     -1.507e-04  2.040e-04  -0.739 0.459889    
## stat144      2.282e-04  2.028e-04   1.126 0.260361    
## stat145     -1.627e-05  2.056e-04  -0.079 0.936917    
## stat146     -6.365e-04  2.052e-04  -3.102 0.001936 ** 
## stat147     -2.574e-04  2.051e-04  -1.255 0.209494    
## stat148     -3.111e-04  2.020e-04  -1.540 0.123600    
## stat149     -6.283e-04  2.049e-04  -3.067 0.002174 ** 
## stat150     -3.922e-05  2.059e-04  -0.190 0.848960    
## stat151      5.189e-04  2.049e-04   2.532 0.011365 *  
## stat152     -1.232e-04  2.016e-04  -0.611 0.541108    
## stat153      3.557e-04  2.064e-04   1.723 0.084963 .  
## stat154      1.474e-04  2.060e-04   0.716 0.474230    
## stat155      1.481e-04  2.024e-04   0.732 0.464181    
## stat156      1.878e-04  2.058e-04   0.912 0.361703    
## stat157     -7.195e-06  2.032e-04  -0.035 0.971752    
## stat158      1.497e-04  2.054e-04   0.729 0.466216    
## stat159      9.139e-05  2.027e-04   0.451 0.652086    
## stat160     -3.256e-05  2.053e-04  -0.159 0.873974    
## stat161      3.144e-04  2.041e-04   1.541 0.123473    
## stat162      5.708e-05  2.015e-04   0.283 0.777005    
## stat163      2.560e-04  2.063e-04   1.241 0.214666    
## stat164      1.136e-04  2.058e-04   0.552 0.581095    
## stat165     -4.807e-05  2.026e-04  -0.237 0.812460    
## stat166     -1.726e-04  2.012e-04  -0.858 0.390961    
## stat167     -5.178e-04  2.041e-04  -2.537 0.011209 *  
## stat168     -2.068e-04  2.024e-04  -1.022 0.306887    
## stat169      1.074e-06  2.035e-04   0.005 0.995789    
## stat170     -1.562e-04  2.038e-04  -0.766 0.443623    
## stat171     -2.270e-04  2.054e-04  -1.105 0.269219    
## stat172      5.966e-04  2.007e-04   2.972 0.002974 ** 
## stat173     -7.250e-05  2.050e-04  -0.354 0.723549    
## stat174      2.123e-04  2.039e-04   1.042 0.297641    
## stat175     -3.867e-04  2.032e-04  -1.903 0.057041 .  
## stat176      4.736e-05  2.036e-04   0.233 0.816074    
## stat177     -3.796e-04  2.039e-04  -1.861 0.062741 .  
## stat178     -1.706e-05  2.061e-04  -0.083 0.934019    
## stat179     -6.416e-05  2.023e-04  -0.317 0.751161    
## stat180     -1.692e-04  2.030e-04  -0.834 0.404486    
## stat181      4.094e-04  2.041e-04   2.006 0.044952 *  
## stat182      2.190e-04  2.047e-04   1.070 0.284685    
## stat183      2.256e-04  2.030e-04   1.112 0.266388    
## stat184      3.365e-04  2.044e-04   1.646 0.099795 .  
## stat185     -2.896e-05  2.013e-04  -0.144 0.885581    
## stat186      1.511e-04  2.053e-04   0.736 0.461882    
## stat187     -1.727e-04  2.032e-04  -0.850 0.395461    
## stat188      2.991e-04  2.035e-04   1.470 0.141742    
## stat189     -6.504e-05  2.057e-04  -0.316 0.751902    
## stat190     -1.081e-04  2.026e-04  -0.533 0.593752    
## stat191     -2.952e-04  2.042e-04  -1.446 0.148312    
## stat192      5.473e-05  2.069e-04   0.264 0.791434    
## stat193      1.205e-04  2.059e-04   0.585 0.558281    
## stat194     -5.902e-06  2.032e-04  -0.029 0.976824    
## stat195      3.088e-04  2.037e-04   1.516 0.129609    
## stat196     -3.806e-04  2.062e-04  -1.846 0.064992 .  
## stat197     -2.438e-04  2.010e-04  -1.213 0.225335    
## stat198     -1.766e-04  2.051e-04  -0.861 0.389140    
## stat199      1.977e-04  2.021e-04   0.979 0.327864    
## stat200     -2.513e-04  2.025e-04  -1.241 0.214686    
## stat201      7.178e-05  2.026e-04   0.354 0.723093    
## stat202      5.806e-06  2.064e-04   0.028 0.977560    
## stat203     -8.281e-05  2.030e-04  -0.408 0.683406    
## stat204     -1.970e-04  2.028e-04  -0.971 0.331552    
## stat205      1.459e-04  2.017e-04   0.723 0.469469    
## stat206     -1.575e-04  2.045e-04  -0.770 0.441276    
## stat207      4.263e-04  2.037e-04   2.093 0.036400 *  
## stat208      7.375e-05  2.044e-04   0.361 0.718265    
## stat209      5.482e-05  2.026e-04   0.271 0.786753    
## stat210     -4.652e-04  2.050e-04  -2.269 0.023318 *  
## stat211     -1.620e-04  2.046e-04  -0.792 0.428394    
## stat212     -3.511e-05  2.043e-04  -0.172 0.863578    
## stat213     -2.529e-04  2.043e-04  -1.238 0.215662    
## stat214     -9.808e-05  2.044e-04  -0.480 0.631401    
## stat215     -2.294e-04  2.044e-04  -1.122 0.261828    
## stat216     -1.732e-05  2.041e-04  -0.085 0.932373    
## stat217      1.871e-05  2.036e-04   0.092 0.926776    
## x18.sqrt     2.643e-02  7.718e-04  34.246  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.02507 on 5053 degrees of freedom
## Multiple R-squared:  0.3722, Adjusted R-squared:  0.3424 
## F-statistic: 12.48 on 240 and 5053 DF,  p-value: < 2.2e-16

cd.full2 = plot.diagnostics(model.full2, data.train2)

## [1] "Number of data points that have Cook's D > 4/n: 267"
## [1] "Number of data points that have Cook's D > 1: 0"

# much more normal residuals than before. 
# Checking to see if distributions are different and if so whcih variables
# High Leverage Plot 
plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,target=one_of(label.names))

ggplot(data=plotData, aes(x=type,y=target)) +
  geom_boxplot(fill='light blue',outlier.shape=NA) +
  scale_y_continuous(name="Target Variable Values",label=scales::comma_format(accuracy=.1)) +
  theme_light() +
  ggtitle('Distribution of High Leverage Points and Normal  Points')

# 2 sample t-tests

plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,one_of(feature.names))

comp.test = lapply(dplyr::select(plotData, one_of(feature.names))
                   , function(x) t.test(x ~ plotData$type, var.equal = TRUE)) 

sig.comp = list.filter(comp.test, p.value < 0.05)
sapply(sig.comp, function(x) x[['p.value']])

##           x4          x14       stat22       stat67       stat74       stat82       stat85       stat95       stat98 
## 1.272018e-02 2.501255e-02 4.594164e-02 3.146324e-02 3.462500e-02 2.956704e-02 1.632652e-02 4.422400e-02 5.613321e-05 
##      stat110      stat128      stat146      stat155      stat186 
## 7.445871e-04 1.127701e-04 2.322952e-02 2.116008e-02 2.106983e-02

mm = melt(plotData, id=c('type')) %>% filter(variable %in% names(sig.comp))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=5, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

# Distribution (box) Plots
mm = melt(plotData, id=c('type'))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=8, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

Grand Means Model

model.null = lm(grand.mean.formula, data.train)
summary(model.null)

## 
## Call:
## lm(formula = grand.mean.formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.115038 -0.023938 -0.003437  0.020538  0.190275 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 2.0969137  0.0004839    4333   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03616 on 5583 degrees of freedom

Variable Selection

Basic: http://www.stat.columbia.edu/~martin/W2024/R10.pdf Cross Validation + Other Metrics: http://www.sthda.com/english/articles/37-model-selection-essentials-in-r/154-stepwise-regression-essentials-in-r/

Forward Selection with CV

Train

if (algo.forward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   , data = data.train
                                   , method = "leapForward"
                                   , feature.names = feature.names)
  model.forward = returned$model
  id = returned$id
}

## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 7 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03425332 0.1038025 0.02663970 0.0006637466 0.02345731 0.0004078960
## 2       2 0.03345274 0.1456791 0.02595410 0.0006744402 0.02299972 0.0005397020
## 3       3 0.03287356 0.1755708 0.02542164 0.0007428313 0.03453720 0.0006065015
## 4       4 0.03238568 0.2002338 0.02476762 0.0007588086 0.03652502 0.0005318967
## 5       5 0.03210370 0.2138463 0.02455180 0.0007501264 0.03860748 0.0004926778
## 6       6 0.03207281 0.2152005 0.02457183 0.0007522223 0.03774776 0.0004857396
## 7       7 0.03198024 0.2196041 0.02449315 0.0007733914 0.03745117 0.0004793219
## 8       8 0.03202042 0.2177381 0.02451834 0.0007986437 0.03807747 0.0004689433
## 9       9 0.03202975 0.2172467 0.02453799 0.0008229237 0.03807716 0.0004777837
## 10     10 0.03204470 0.2166573 0.02458335 0.0008692181 0.03928587 0.0004970027
## 11     11 0.03205450 0.2163469 0.02459478 0.0008900320 0.04095855 0.0005215186
## 12     12 0.03202978 0.2176650 0.02459328 0.0009083095 0.04241137 0.0005532378
## 13     13 0.03203789 0.2172484 0.02458580 0.0009255880 0.04316006 0.0005482828
## 14     14 0.03201746 0.2181256 0.02456640 0.0009074620 0.04320969 0.0005352318
## 15     15 0.03200221 0.2187415 0.02456541 0.0008875529 0.04192161 0.0005198344
## 16     16 0.03200929 0.2183536 0.02456633 0.0008841716 0.04062908 0.0005175875
## 17     17 0.03200941 0.2183028 0.02457658 0.0008754303 0.04056618 0.0005016424
## 18     18 0.03200735 0.2184496 0.02456975 0.0008681755 0.04123859 0.0005068268
## 19     19 0.03201551 0.2181085 0.02457413 0.0008570310 0.04119713 0.0005148643
## 20     20 0.03202603 0.2176078 0.02457884 0.0008480833 0.03917978 0.0005005236
## 21     21 0.03202632 0.2177041 0.02459583 0.0008693105 0.04074689 0.0005038274
## 22     22 0.03204354 0.2169746 0.02461131 0.0008564701 0.04124006 0.0004985774
## 23     23 0.03206535 0.2160674 0.02462354 0.0008844102 0.04161250 0.0005130282
## 24     24 0.03207994 0.2154113 0.02464695 0.0008772836 0.04207940 0.0005196965
## 25     25 0.03209704 0.2147090 0.02465619 0.0008773609 0.04228189 0.0005116772
## 26     26 0.03211100 0.2140913 0.02466218 0.0008659019 0.04225409 0.0005187658
## 27     27 0.03212285 0.2136133 0.02466584 0.0008827717 0.04270849 0.0005242503
## 28     28 0.03215456 0.2121861 0.02469439 0.0008725705 0.04224414 0.0005156718
## 29     29 0.03218762 0.2107482 0.02470955 0.0008787025 0.04248352 0.0005324064
## 30     30 0.03218762 0.2107042 0.02470595 0.0008650116 0.04107858 0.0005241851
## 31     31 0.03220642 0.2098019 0.02472348 0.0008464269 0.04015817 0.0005140751
## 32     32 0.03222899 0.2087787 0.02474004 0.0008297812 0.03929632 0.0004989500
## 33     33 0.03225221 0.2077077 0.02476354 0.0008412632 0.03917881 0.0005148373
## 34     34 0.03225396 0.2076823 0.02476104 0.0008400931 0.03914337 0.0005024854
## 35     35 0.03225068 0.2078406 0.02476025 0.0008281323 0.03920479 0.0005063259
## 36     36 0.03226815 0.2070115 0.02477967 0.0008216528 0.03891611 0.0004838645
## 37     37 0.03226428 0.2072522 0.02477139 0.0008323671 0.03936488 0.0005022177
## 38     38 0.03226734 0.2072163 0.02476483 0.0008578724 0.03926701 0.0005138741
## 39     39 0.03228366 0.2064904 0.02477378 0.0008559894 0.03898850 0.0004998477
## 40     40 0.03227727 0.2067550 0.02475892 0.0008470391 0.03845076 0.0005151089
## 41     41 0.03227475 0.2068510 0.02476785 0.0008320415 0.03850472 0.0005087222
## 42     42 0.03227794 0.2066541 0.02476377 0.0008265100 0.03795841 0.0004964451
## 43     43 0.03229750 0.2057985 0.02477069 0.0008207917 0.03771944 0.0004914882
## 44     44 0.03230157 0.2057027 0.02477333 0.0008188554 0.03759670 0.0004896224
## 45     45 0.03230461 0.2056555 0.02476923 0.0008094334 0.03737533 0.0004756162
## 46     46 0.03230912 0.2055744 0.02476657 0.0008284662 0.03819449 0.0004832617
## 47     47 0.03231195 0.2054554 0.02476226 0.0008261027 0.03801366 0.0004963263
## 48     48 0.03231987 0.2051875 0.02477140 0.0008416278 0.03820537 0.0005113370
## 49     49 0.03231393 0.2055364 0.02475981 0.0008402889 0.03778873 0.0005245987
## 50     50 0.03231931 0.2053657 0.02476205 0.0008294511 0.03754007 0.0005273374
## 51     51 0.03231760 0.2054704 0.02475627 0.0008215351 0.03726838 0.0005238694
## 52     52 0.03231577 0.2056270 0.02475740 0.0008311694 0.03793016 0.0005298481
## 53     53 0.03231391 0.2057282 0.02474924 0.0008346159 0.03816937 0.0005342212
## 54     54 0.03232302 0.2053976 0.02475217 0.0008530000 0.03846713 0.0005383056
## 55     55 0.03231759 0.2056344 0.02474141 0.0008600879 0.03845304 0.0005507192
## 56     56 0.03231438 0.2058017 0.02473980 0.0008603944 0.03828008 0.0005559409
## 57     57 0.03232896 0.2051983 0.02475823 0.0008682246 0.03916968 0.0005667832
## 58     58 0.03234246 0.2046616 0.02476647 0.0008781541 0.03945602 0.0005785650
## 59     59 0.03234400 0.2046528 0.02476576 0.0008928999 0.03980144 0.0005940435
## 60     60 0.03233157 0.2052024 0.02475111 0.0008843889 0.03964404 0.0005846254
## 61     61 0.03234112 0.2048464 0.02476157 0.0008801030 0.03977207 0.0005808083
## 62     62 0.03234582 0.2047093 0.02477684 0.0008849383 0.03987738 0.0005884327
## 63     63 0.03233759 0.2050896 0.02477178 0.0008901416 0.04016549 0.0006084873
## 64     64 0.03234720 0.2046886 0.02477744 0.0008939146 0.04051177 0.0006085268
## 65     65 0.03234535 0.2047567 0.02478268 0.0008951426 0.04023932 0.0006137211
## 66     66 0.03235396 0.2043840 0.02479212 0.0008932029 0.03986344 0.0006145574
## 67     67 0.03234908 0.2046800 0.02478327 0.0009111084 0.04079861 0.0006304013
## 68     68 0.03234661 0.2047798 0.02478415 0.0009167986 0.04101754 0.0006319890
## 69     69 0.03234990 0.2046026 0.02478412 0.0009082548 0.04065113 0.0006179838
## 70     70 0.03234394 0.2048595 0.02478127 0.0008989999 0.04005898 0.0006191626
## 71     71 0.03234317 0.2049899 0.02477944 0.0009116507 0.04057801 0.0006319947
## 72     72 0.03234218 0.2050562 0.02477725 0.0009225632 0.04092472 0.0006289144
## 73     73 0.03234321 0.2050800 0.02477431 0.0009247104 0.04115610 0.0006401351
## 74     74 0.03234830 0.2048629 0.02477738 0.0009335101 0.04125402 0.0006462124
## 75     75 0.03234907 0.2047908 0.02477024 0.0009311665 0.04133942 0.0006425680
## 76     76 0.03234353 0.2050833 0.02475716 0.0009408045 0.04141778 0.0006444117
## 77     77 0.03235695 0.2045206 0.02476191 0.0009483779 0.04159505 0.0006463661
## 78     78 0.03235124 0.2047769 0.02475875 0.0009368024 0.04155216 0.0006278327
## 79     79 0.03233991 0.2052514 0.02474672 0.0009356728 0.04128415 0.0006253143
## 80     80 0.03233935 0.2052502 0.02474342 0.0009364226 0.04046615 0.0006293013
## 81     81 0.03235140 0.2047529 0.02474696 0.0009392787 0.04025579 0.0006282085
## 82     82 0.03234990 0.2048621 0.02474915 0.0009437216 0.04040662 0.0006406448
## 83     83 0.03235093 0.2048937 0.02474889 0.0009465195 0.04084632 0.0006503105
## 84     84 0.03235277 0.2048314 0.02475204 0.0009485554 0.04069506 0.0006572867
## 85     85 0.03235186 0.2048757 0.02475047 0.0009434288 0.04045353 0.0006524788
## 86     86 0.03235746 0.2046309 0.02475756 0.0009416367 0.04042292 0.0006549261
## 87     87 0.03236327 0.2043201 0.02476534 0.0009416268 0.04028398 0.0006608134
## 88     88 0.03237567 0.2038072 0.02478221 0.0009460851 0.04028672 0.0006616953
## 89     89 0.03237791 0.2037248 0.02478741 0.0009466909 0.04026254 0.0006586678
## 90     90 0.03238585 0.2033874 0.02479508 0.0009464895 0.04006450 0.0006524957
## 91     91 0.03238878 0.2032821 0.02480052 0.0009556147 0.04007944 0.0006638729
## 92     92 0.03238919 0.2032675 0.02480020 0.0009579794 0.03982103 0.0006570841
## 93     93 0.03239792 0.2029131 0.02480727 0.0009587934 0.04005396 0.0006601700
## 94     94 0.03241220 0.2022693 0.02481437 0.0009455669 0.03924290 0.0006530140
## 95     95 0.03241536 0.2020674 0.02482375 0.0009355865 0.03853373 0.0006376410
## 96     96 0.03242144 0.2017556 0.02483308 0.0009295837 0.03811159 0.0006375916
## 97     97 0.03243073 0.2013651 0.02484734 0.0009240696 0.03792972 0.0006410813
## 98     98 0.03242584 0.2016171 0.02484847 0.0009239515 0.03811274 0.0006413954
## 99     99 0.03242475 0.2016583 0.02485208 0.0009269770 0.03810114 0.0006484059
## 100   100 0.03242597 0.2015645 0.02485265 0.0009258770 0.03789101 0.0006452211
## 101   101 0.03242645 0.2015480 0.02485978 0.0009285266 0.03793818 0.0006485322
## 102   102 0.03242781 0.2014810 0.02486084 0.0009218449 0.03774888 0.0006443989
## 103   103 0.03242466 0.2016116 0.02485947 0.0009226232 0.03754371 0.0006416607
## 104   104 0.03242321 0.2016842 0.02485329 0.0009254970 0.03725555 0.0006363103
## 105   105 0.03243106 0.2013468 0.02485658 0.0009270190 0.03742344 0.0006426924
## 106   106 0.03242832 0.2015017 0.02485248 0.0009219362 0.03753943 0.0006345397
## 107   107 0.03243376 0.2013307 0.02485344 0.0009225388 0.03782821 0.0006398205
## 108   108 0.03243638 0.2012358 0.02485895 0.0009203937 0.03774353 0.0006338635
## 109   109 0.03243530 0.2013067 0.02485823 0.0009262451 0.03799016 0.0006355999
## 110   110 0.03243352 0.2013729 0.02485929 0.0009232081 0.03777778 0.0006267942
## 111   111 0.03244225 0.2009967 0.02486622 0.0009208366 0.03734784 0.0006276097
## 112   112 0.03244543 0.2008877 0.02486814 0.0009240197 0.03753100 0.0006271282
## 113   113 0.03243398 0.2013656 0.02486111 0.0009205469 0.03729800 0.0006193721
## 114   114 0.03243300 0.2014043 0.02485969 0.0009231327 0.03727070 0.0006196617
## 115   115 0.03244053 0.2010777 0.02486385 0.0009160425 0.03711190 0.0006095460
## 116   116 0.03244156 0.2010298 0.02486710 0.0009164051 0.03697066 0.0006065728
## 117   117 0.03243880 0.2011363 0.02487113 0.0009138376 0.03685727 0.0006070892
## 118   118 0.03243880 0.2011905 0.02487369 0.0009178758 0.03700262 0.0006196934
## 119   119 0.03244007 0.2011457 0.02487416 0.0009141491 0.03685142 0.0006225413
## 120   120 0.03243249 0.2014528 0.02487547 0.0009035198 0.03654021 0.0006124608
## 121   121 0.03243858 0.2012635 0.02488380 0.0009086119 0.03668680 0.0006170326
## 122   122 0.03243512 0.2014345 0.02488043 0.0009064722 0.03656364 0.0006117069
## 123   123 0.03243587 0.2013802 0.02487860 0.0009058547 0.03638222 0.0006088540
## 124   124 0.03243583 0.2014097 0.02488153 0.0009010734 0.03623152 0.0006107263
## 125   125 0.03243313 0.2015543 0.02487929 0.0009049914 0.03649795 0.0006085321
## 126   126 0.03243163 0.2016280 0.02488041 0.0009065971 0.03659907 0.0006102515
## 127   127 0.03243668 0.2014286 0.02488789 0.0009045333 0.03639173 0.0006050556
## 128   128 0.03243092 0.2016512 0.02488153 0.0009028268 0.03618705 0.0006036632
## 129   129 0.03243757 0.2013461 0.02488368 0.0009002642 0.03601815 0.0005985909
## 130   130 0.03243637 0.2013482 0.02488584 0.0008960741 0.03578918 0.0005920354
## 131   131 0.03243833 0.2012623 0.02488550 0.0008963951 0.03570313 0.0005951867
## 132   132 0.03244517 0.2009284 0.02489332 0.0008901719 0.03533913 0.0005906992
## 133   133 0.03244870 0.2007981 0.02489772 0.0008902826 0.03516049 0.0005834044
## 134   134 0.03245370 0.2005783 0.02489999 0.0008963292 0.03504266 0.0005901542
## 135   135 0.03245767 0.2004205 0.02490864 0.0008970495 0.03515772 0.0005985022
## 136   136 0.03247089 0.1998588 0.02492264 0.0008997920 0.03520020 0.0006001149
## 137   137 0.03247872 0.1995291 0.02492926 0.0009026981 0.03537158 0.0005986826
## 138   138 0.03247557 0.1996855 0.02492268 0.0008993834 0.03537595 0.0005953179
## 139   139 0.03247701 0.1996537 0.02492970 0.0008999057 0.03567589 0.0006017647
## 140   140 0.03248245 0.1994570 0.02493004 0.0008955063 0.03582755 0.0005989570
## 141   141 0.03247939 0.1996032 0.02492411 0.0008980332 0.03565632 0.0006035756
## 142   142 0.03248172 0.1995321 0.02492472 0.0008976698 0.03570799 0.0005974015
## 143   143 0.03248314 0.1994743 0.02492633 0.0008966188 0.03565448 0.0005953337
## 144   144 0.03248984 0.1991671 0.02493386 0.0008900063 0.03552741 0.0005917431
## 145   145 0.03249422 0.1990028 0.02493739 0.0008911373 0.03563544 0.0005885409
## 146   146 0.03249049 0.1991740 0.02493373 0.0008921655 0.03557022 0.0005936148
## 147   147 0.03248872 0.1992403 0.02493470 0.0008909599 0.03550745 0.0005947929
## 148   148 0.03249173 0.1990908 0.02493564 0.0008874795 0.03537123 0.0005906702
## 149   149 0.03248418 0.1994458 0.02492668 0.0008885112 0.03555083 0.0005933954
## 150   150 0.03248640 0.1993665 0.02492529 0.0008928167 0.03570149 0.0005955367
## 151   151 0.03248489 0.1994358 0.02492698 0.0008878432 0.03552644 0.0005925543
## 152   152 0.03248049 0.1996195 0.02492333 0.0008856623 0.03536552 0.0005901828
## 153   153 0.03247503 0.1998757 0.02491768 0.0008855359 0.03556416 0.0005870722
## 154   154 0.03247069 0.2000737 0.02491646 0.0008849520 0.03566817 0.0005848079
## 155   155 0.03247046 0.2000911 0.02491222 0.0008776927 0.03557668 0.0005777091
## 156   156 0.03247176 0.2000522 0.02491193 0.0008778872 0.03560527 0.0005828458
## 157   157 0.03247203 0.2000206 0.02490786 0.0008800305 0.03541244 0.0005833027
## 158   158 0.03247718 0.1997660 0.02490960 0.0008772766 0.03509925 0.0005819353
## 159   159 0.03247867 0.1997085 0.02490844 0.0008815291 0.03503918 0.0005860433
## 160   160 0.03248293 0.1995280 0.02491104 0.0008814234 0.03493910 0.0005853529
## 161   161 0.03248281 0.1995462 0.02491118 0.0008793696 0.03479915 0.0005799937
## 162   162 0.03248364 0.1995048 0.02491114 0.0008744203 0.03465827 0.0005792231
## 163   163 0.03248621 0.1994077 0.02491516 0.0008739026 0.03462654 0.0005826598
## 164   164 0.03248719 0.1993687 0.02491684 0.0008751988 0.03447777 0.0005808543
## 165   165 0.03248961 0.1992467 0.02492059 0.0008737361 0.03436242 0.0005782947
## 166   166 0.03248975 0.1992464 0.02492211 0.0008738415 0.03449333 0.0005731866
## 167   167 0.03248746 0.1993610 0.02491951 0.0008738797 0.03462349 0.0005751304
## 168   168 0.03248810 0.1993495 0.02491969 0.0008847457 0.03497328 0.0005832000
## 169   169 0.03249007 0.1992913 0.02491946 0.0008833885 0.03502697 0.0005822244
## 170   170 0.03248963 0.1993095 0.02491798 0.0008847103 0.03507500 0.0005854299
## 171   171 0.03248493 0.1995119 0.02491398 0.0008835991 0.03504001 0.0005862350
## 172   172 0.03248464 0.1995279 0.02491215 0.0008804676 0.03495792 0.0005849363
## 173   173 0.03248417 0.1995409 0.02491196 0.0008764807 0.03481496 0.0005801069
## 174   174 0.03248538 0.1994929 0.02491369 0.0008743601 0.03475987 0.0005799823
## 175   175 0.03248319 0.1995850 0.02491114 0.0008750088 0.03476820 0.0005796773
## 176   176 0.03248338 0.1996087 0.02491118 0.0008730968 0.03493957 0.0005772947
## 177   177 0.03248246 0.1996394 0.02491046 0.0008707840 0.03484665 0.0005784407
## 178   178 0.03248466 0.1995477 0.02491277 0.0008705731 0.03479142 0.0005803902
## 179   179 0.03248741 0.1994256 0.02491312 0.0008704373 0.03466812 0.0005782668
## 180   180 0.03248611 0.1994708 0.02491388 0.0008693338 0.03480152 0.0005798784
## 181   181 0.03248536 0.1994901 0.02491387 0.0008697065 0.03465894 0.0005776697
## 182   182 0.03248267 0.1996061 0.02491199 0.0008683756 0.03455420 0.0005750150
## 183   183 0.03248412 0.1995416 0.02491447 0.0008700606 0.03462214 0.0005780112
## 184   184 0.03248361 0.1995765 0.02491433 0.0008734609 0.03466139 0.0005789079
## 185   185 0.03248646 0.1994422 0.02491713 0.0008753482 0.03451757 0.0005765737
## 186   186 0.03248448 0.1995271 0.02491464 0.0008747700 0.03450348 0.0005784177
## 187   187 0.03248309 0.1995821 0.02491304 0.0008750297 0.03448864 0.0005776661
## 188   188 0.03248527 0.1994988 0.02491386 0.0008785071 0.03462768 0.0005825981
## 189   189 0.03248500 0.1994964 0.02491444 0.0008810888 0.03462785 0.0005837340
## 190   190 0.03248349 0.1995651 0.02491263 0.0008812024 0.03465382 0.0005826084
## 191   191 0.03248518 0.1995021 0.02491375 0.0008807648 0.03467276 0.0005838545
## 192   192 0.03248612 0.1994681 0.02491460 0.0008796605 0.03469117 0.0005832579
## 193   193 0.03248668 0.1994614 0.02491517 0.0008797943 0.03485550 0.0005869625
## 194   194 0.03249018 0.1993214 0.02491512 0.0008808198 0.03495191 0.0005894243
## 195   195 0.03248991 0.1993525 0.02491502 0.0008801786 0.03502950 0.0005905530
## 196   196 0.03249411 0.1991847 0.02491980 0.0008784703 0.03503171 0.0005888367
## 197   197 0.03249717 0.1990595 0.02492232 0.0008766760 0.03488136 0.0005876382
## 198   198 0.03249700 0.1990701 0.02492205 0.0008757393 0.03486623 0.0005874632
## 199   199 0.03249606 0.1991095 0.02492113 0.0008768510 0.03483606 0.0005869788
## 200   200 0.03249804 0.1990300 0.02492282 0.0008790443 0.03494330 0.0005911075
## 201   201 0.03249918 0.1989736 0.02492359 0.0008772378 0.03478190 0.0005879876
## 202   202 0.03249925 0.1989797 0.02492386 0.0008773952 0.03476409 0.0005909456
## 203   203 0.03249868 0.1990042 0.02492344 0.0008746218 0.03467848 0.0005862002
## 204   204 0.03249710 0.1990668 0.02492457 0.0008767562 0.03466324 0.0005860671
## 205   205 0.03249818 0.1990258 0.02492668 0.0008780947 0.03469432 0.0005871516
## 206   206 0.03249791 0.1990267 0.02492721 0.0008773839 0.03471409 0.0005887559
## 207   207 0.03249862 0.1990058 0.02492790 0.0008775362 0.03474592 0.0005908043
## 208   208 0.03249686 0.1990829 0.02492739 0.0008758180 0.03471911 0.0005886337
## 209   209 0.03249806 0.1990351 0.02492811 0.0008743648 0.03465560 0.0005880437
## 210   210 0.03249678 0.1990961 0.02492827 0.0008739620 0.03463195 0.0005884302
## 211   211 0.03249666 0.1990965 0.02492926 0.0008748057 0.03466463 0.0005888852
## 212   212 0.03249713 0.1990703 0.02492857 0.0008751715 0.03464105 0.0005900112
## 213   213 0.03249735 0.1990601 0.02492847 0.0008754028 0.03463836 0.0005904566
## 214   214 0.03249683 0.1990798 0.02492931 0.0008746378 0.03454403 0.0005900086
## 215   215 0.03249741 0.1990594 0.02492879 0.0008751510 0.03453079 0.0005909715
## 216   216 0.03249792 0.1990439 0.02492824 0.0008752641 0.03452670 0.0005917436
## 217   217 0.03249738 0.1990614 0.02492745 0.0008752139 0.03450778 0.0005904128
## 218   218 0.03249877 0.1990066 0.02492862 0.0008757781 0.03449711 0.0005906042
## 219   219 0.03249911 0.1989887 0.02492877 0.0008756685 0.03447755 0.0005902879
## 220   220 0.03249969 0.1989647 0.02492930 0.0008755146 0.03444921 0.0005898006
## 221   221 0.03249966 0.1989690 0.02492888 0.0008759477 0.03447460 0.0005896631
## 222   222 0.03249987 0.1989670 0.02492902 0.0008754698 0.03450162 0.0005897607
## 223   223 0.03249982 0.1989739 0.02492932 0.0008756809 0.03452805 0.0005894229
## 224   224 0.03250048 0.1989442 0.02492961 0.0008750176 0.03448473 0.0005889142
## 225   225 0.03250152 0.1989007 0.02492986 0.0008742688 0.03447501 0.0005895298
## 226   226 0.03250154 0.1988985 0.02493019 0.0008751578 0.03452779 0.0005899196
## 227   227 0.03250120 0.1989146 0.02493008 0.0008746615 0.03453660 0.0005894590
## 228   228 0.03250117 0.1989154 0.02493019 0.0008744490 0.03450365 0.0005893487
## 229   229 0.03250138 0.1989092 0.02493076 0.0008748681 0.03450909 0.0005898006
## 230   230 0.03250202 0.1988837 0.02493127 0.0008753845 0.03451525 0.0005907059
## 231   231 0.03250174 0.1988955 0.02493155 0.0008752620 0.03453263 0.0005906768
## 232   232 0.03250172 0.1988959 0.02493129 0.0008753881 0.03455645 0.0005909734
## 233   233 0.03250199 0.1988857 0.02493163 0.0008752934 0.03453992 0.0005910003
## 234   234 0.03250185 0.1988908 0.02493129 0.0008752629 0.03453640 0.0005907534
## 235   235 0.03250217 0.1988765 0.02493170 0.0008753765 0.03453248 0.0005912316
## 236   236 0.03250231 0.1988701 0.02493181 0.0008753587 0.03452230 0.0005914375
## 237   237 0.03250231 0.1988711 0.02493180 0.0008753858 0.03452655 0.0005915679
## 238   238 0.03250246 0.1988655 0.02493188 0.0008753904 0.03452419 0.0005916808
## 239   239 0.03250247 0.1988651 0.02493187 0.0008753765 0.03452171 0.0005917486
## 240   240 0.03250253 0.1988622 0.02493193 0.0008752879 0.03451534 0.0005917198
## [1] "Best Model"
##   nvmax
## 7     7

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.010989e+00  2.005707e+00  2.016270e+00
## x4          -4.399973e-05 -6.159867e-05 -2.640079e-05
## x7           1.108087e-02  9.833384e-03  1.232835e-02
## x9           3.310416e-03  2.662524e-03  3.958309e-03
## x17          1.507310e-03  8.723715e-04  2.142248e-03
## stat98       3.548406e-03  3.072152e-03  4.024661e-03
## stat110     -3.282208e-03 -3.767312e-03 -2.797105e-03
## x18.sqrt     2.527794e-02  2.342698e-02  2.712889e-02

Test

if (algo.forward.caret == TRUE){
    test.model(model=model.forward, test=data.test
             ,method = 'leapForward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}

## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.044   2.084   2.097   2.097   2.109   2.147 
## [1] "leapForward  Test MSE: 0.000942359887602509"

Backward Elimination with CV

Train

if (algo.backward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapBackward"
                                   ,feature.names =  feature.names)
  model.backward = returned$model
  id = returned$id
}

## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 7 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03425332 0.1038025 0.02663970 0.0006637466 0.02345731 0.0004078960
## 2       2 0.03345274 0.1456791 0.02595410 0.0006744402 0.02299972 0.0005397020
## 3       3 0.03287356 0.1755708 0.02542164 0.0007428313 0.03453720 0.0006065015
## 4       4 0.03238568 0.2002338 0.02476762 0.0007588086 0.03652502 0.0005318967
## 5       5 0.03210370 0.2138463 0.02455180 0.0007501264 0.03860748 0.0004926778
## 6       6 0.03207281 0.2152005 0.02457183 0.0007522223 0.03774776 0.0004857396
## 7       7 0.03198024 0.2196041 0.02449315 0.0007733914 0.03745117 0.0004793219
## 8       8 0.03202042 0.2177381 0.02451834 0.0007986437 0.03807747 0.0004689433
## 9       9 0.03202975 0.2172467 0.02453799 0.0008229237 0.03807716 0.0004777837
## 10     10 0.03204470 0.2166573 0.02458335 0.0008692181 0.03928587 0.0004970027
## 11     11 0.03205450 0.2163469 0.02459478 0.0008900320 0.04095855 0.0005215186
## 12     12 0.03202978 0.2176650 0.02459328 0.0009083095 0.04241137 0.0005532378
## 13     13 0.03203789 0.2172484 0.02458580 0.0009255880 0.04316006 0.0005482828
## 14     14 0.03201746 0.2181256 0.02456640 0.0009074620 0.04320969 0.0005352318
## 15     15 0.03200221 0.2187415 0.02456541 0.0008875529 0.04192161 0.0005198344
## 16     16 0.03200929 0.2183536 0.02456633 0.0008841716 0.04062908 0.0005175875
## 17     17 0.03201623 0.2179853 0.02457879 0.0008811756 0.04097910 0.0005060063
## 18     18 0.03201548 0.2181183 0.02457520 0.0008755563 0.04167981 0.0005176730
## 19     19 0.03201459 0.2181521 0.02457154 0.0008561828 0.04113889 0.0005097624
## 20     20 0.03202603 0.2176078 0.02457884 0.0008480833 0.03917978 0.0005005236
## 21     21 0.03202865 0.2176383 0.02459647 0.0008657724 0.04060881 0.0005032706
## 22     22 0.03204473 0.2169285 0.02459972 0.0008469533 0.04085003 0.0004937527
## 23     23 0.03206098 0.2162634 0.02460930 0.0008841281 0.04174735 0.0005099297
## 24     24 0.03208911 0.2149458 0.02463996 0.0008641596 0.04142237 0.0005120474
## 25     25 0.03210544 0.2142768 0.02465212 0.0008646829 0.04140997 0.0005120867
## 26     26 0.03212017 0.2137234 0.02466389 0.0008634894 0.04206252 0.0005121214
## 27     27 0.03213479 0.2130040 0.02466578 0.0008519501 0.04047248 0.0005036940
## 28     28 0.03216692 0.2116068 0.02469126 0.0008543654 0.04077600 0.0005076495
## 29     29 0.03218570 0.2107178 0.02470522 0.0008509819 0.04080542 0.0005245350
## 30     30 0.03219028 0.2104525 0.02469786 0.0008407803 0.03991562 0.0005158468
## 31     31 0.03221768 0.2092514 0.02472540 0.0008349268 0.03976746 0.0005129946
## 32     32 0.03223220 0.2086308 0.02474066 0.0008400335 0.04001872 0.0005083977
## 33     33 0.03223634 0.2084468 0.02474432 0.0008228124 0.03922738 0.0004941479
## 34     34 0.03225122 0.2077992 0.02475405 0.0008217371 0.03876795 0.0004893599
## 35     35 0.03225900 0.2074829 0.02475287 0.0008240531 0.03941395 0.0005040681
## 36     36 0.03227544 0.2067578 0.02477485 0.0008179787 0.03915783 0.0004770500
## 37     37 0.03225675 0.2077521 0.02474526 0.0008476396 0.04054152 0.0004928211
## 38     38 0.03225720 0.2077773 0.02474213 0.0008406734 0.04016182 0.0005050043
## 39     39 0.03226090 0.2075128 0.02474110 0.0008313780 0.03957226 0.0005042573
## 40     40 0.03227245 0.2069610 0.02475569 0.0008220915 0.03888785 0.0005047208
## 41     41 0.03229145 0.2061642 0.02476215 0.0008071155 0.03882902 0.0004977170
## 42     42 0.03230389 0.2055462 0.02477372 0.0007907677 0.03782580 0.0004763453
## 43     43 0.03229428 0.2060093 0.02476251 0.0007955331 0.03733335 0.0004668917
## 44     44 0.03229774 0.2058901 0.02476817 0.0008039186 0.03743932 0.0004687772
## 45     45 0.03229504 0.2060363 0.02476530 0.0008067735 0.03725978 0.0004629515
## 46     46 0.03230076 0.2059024 0.02475716 0.0008044798 0.03791314 0.0004724821
## 47     47 0.03230440 0.2057873 0.02475499 0.0007930073 0.03740394 0.0004803090
## 48     48 0.03230637 0.2058095 0.02475444 0.0008053698 0.03754047 0.0004912001
## 49     49 0.03230980 0.2057258 0.02475726 0.0007909394 0.03706898 0.0004944410
## 50     50 0.03231186 0.2056870 0.02475429 0.0008001669 0.03702126 0.0004965228
## 51     51 0.03231204 0.2057881 0.02475278 0.0008095080 0.03774652 0.0005116265
## 52     52 0.03230799 0.2059734 0.02474452 0.0008095869 0.03756263 0.0005168704
## 53     53 0.03230719 0.2060785 0.02473988 0.0008280382 0.03822082 0.0005263669
## 54     54 0.03232644 0.2052333 0.02475224 0.0008494328 0.03782618 0.0005381098
## 55     55 0.03231833 0.2056291 0.02474760 0.0008501996 0.03812664 0.0005485853
## 56     56 0.03231958 0.2055521 0.02475384 0.0008495129 0.03788766 0.0005484292
## 57     57 0.03232559 0.2053413 0.02475969 0.0008621589 0.03869276 0.0005631305
## 58     58 0.03233239 0.2051260 0.02476605 0.0008787821 0.03951827 0.0005823639
## 59     59 0.03232655 0.2054135 0.02475957 0.0008923738 0.03978438 0.0005972076
## 60     60 0.03233220 0.2051646 0.02476674 0.0008832325 0.03939827 0.0005872547
## 61     61 0.03233897 0.2049324 0.02478007 0.0008944670 0.03996221 0.0006041239
## 62     62 0.03233688 0.2050736 0.02478691 0.0008979559 0.03994090 0.0006066822
## 63     63 0.03233164 0.2053450 0.02477968 0.0009019015 0.04040472 0.0006099315
## 64     64 0.03234154 0.2049146 0.02478660 0.0009069066 0.04066563 0.0006116384
## 65     65 0.03233338 0.2053073 0.02478218 0.0009141948 0.04082183 0.0006226000
## 66     66 0.03233371 0.2053373 0.02478257 0.0009192748 0.04138004 0.0006298045
## 67     67 0.03234096 0.2050339 0.02478651 0.0009218462 0.04111801 0.0006310186
## 68     68 0.03233542 0.2052236 0.02477734 0.0009051178 0.04012101 0.0006082848
## 69     69 0.03234685 0.2047373 0.02478546 0.0009038962 0.03997937 0.0006183930
## 70     70 0.03234070 0.2050546 0.02478213 0.0009102601 0.04040365 0.0006240110
## 71     71 0.03233957 0.2051443 0.02477904 0.0009153793 0.04078713 0.0006340950
## 72     72 0.03234976 0.2046927 0.02478323 0.0009235398 0.04134618 0.0006473123
## 73     73 0.03235191 0.2046096 0.02478638 0.0009206058 0.04149670 0.0006459423
## 74     74 0.03235144 0.2046963 0.02478104 0.0009313752 0.04154318 0.0006494079
## 75     75 0.03235147 0.2046688 0.02477816 0.0009256366 0.04110153 0.0006353433
## 76     76 0.03236088 0.2043201 0.02477349 0.0009385307 0.04135852 0.0006362904
## 77     77 0.03236037 0.2043797 0.02476188 0.0009521149 0.04167495 0.0006404720
## 78     78 0.03235389 0.2046875 0.02476024 0.0009548929 0.04198551 0.0006442619
## 79     79 0.03235303 0.2047848 0.02475704 0.0009611994 0.04218681 0.0006530337
## 80     80 0.03236731 0.2041524 0.02476343 0.0009562034 0.04179720 0.0006519405
## 81     81 0.03237199 0.2039507 0.02476400 0.0009600472 0.04133866 0.0006626356
## 82     82 0.03235828 0.2045655 0.02476100 0.0009556789 0.04099425 0.0006601910
## 83     83 0.03235923 0.2045730 0.02476208 0.0009557336 0.04134538 0.0006645566
## 84     84 0.03235912 0.2045519 0.02476275 0.0009458908 0.04104955 0.0006556081
## 85     85 0.03235098 0.2048861 0.02474872 0.0009384398 0.04066529 0.0006426720
## 86     86 0.03236183 0.2044052 0.02476559 0.0009437937 0.04062132 0.0006505516
## 87     87 0.03237355 0.2038795 0.02478416 0.0009426925 0.04033215 0.0006592028
## 88     88 0.03237546 0.2038024 0.02479239 0.0009355022 0.03981017 0.0006533088
## 89     89 0.03237998 0.2036086 0.02479028 0.0009429903 0.03972077 0.0006543550
## 90     90 0.03238978 0.2032025 0.02479975 0.0009478051 0.03969487 0.0006506994
## 91     91 0.03238828 0.2033108 0.02480334 0.0009536522 0.04008011 0.0006581045
## 92     92 0.03239534 0.2030323 0.02480906 0.0009524306 0.03992626 0.0006514516
## 93     93 0.03240158 0.2027711 0.02481625 0.0009579399 0.04011574 0.0006548657
## 94     94 0.03241649 0.2020803 0.02482217 0.0009439169 0.03912853 0.0006477592
## 95     95 0.03241972 0.2018991 0.02483142 0.0009411456 0.03838384 0.0006448833
## 96     96 0.03242091 0.2017834 0.02483403 0.0009332973 0.03807143 0.0006407759
## 97     97 0.03242170 0.2017911 0.02484008 0.0009320744 0.03798944 0.0006471388
## 98     98 0.03241496 0.2020941 0.02483746 0.0009370558 0.03830581 0.0006491114
## 99     99 0.03241647 0.2020004 0.02484522 0.0009354085 0.03823085 0.0006526373
## 100   100 0.03241456 0.2020706 0.02484691 0.0009415328 0.03844397 0.0006550772
## 101   101 0.03241994 0.2018548 0.02485389 0.0009382804 0.03830023 0.0006529281
## 102   102 0.03242704 0.2015947 0.02485644 0.0009384459 0.03824000 0.0006535156
## 103   103 0.03242471 0.2016784 0.02485451 0.0009367696 0.03812976 0.0006516684
## 104   104 0.03242053 0.2018569 0.02484714 0.0009326418 0.03790233 0.0006432317
## 105   105 0.03243341 0.2013415 0.02486055 0.0009317135 0.03803062 0.0006472112
## 106   106 0.03244346 0.2008990 0.02486780 0.0009224619 0.03785437 0.0006410351
## 107   107 0.03244066 0.2010790 0.02486146 0.0009281954 0.03824324 0.0006460537
## 108   108 0.03243815 0.2011665 0.02486270 0.0009232100 0.03787161 0.0006415315
## 109   109 0.03243641 0.2012584 0.02486257 0.0009228068 0.03770281 0.0006354267
## 110   110 0.03243476 0.2013342 0.02485780 0.0009208155 0.03772985 0.0006318386
## 111   111 0.03244153 0.2010507 0.02486671 0.0009264927 0.03777319 0.0006347899
## 112   112 0.03244512 0.2008753 0.02487173 0.0009294150 0.03757041 0.0006301293
## 113   113 0.03244228 0.2009608 0.02487220 0.0009251368 0.03737047 0.0006229938
## 114   114 0.03243870 0.2011357 0.02487080 0.0009202326 0.03717711 0.0006153067
## 115   115 0.03244476 0.2008789 0.02487443 0.0009184403 0.03715169 0.0006140191
## 116   116 0.03244728 0.2007760 0.02488196 0.0009179865 0.03700468 0.0006074762
## 117   117 0.03244190 0.2010197 0.02488197 0.0009123737 0.03677916 0.0006016770
## 118   118 0.03244894 0.2007487 0.02488533 0.0009126452 0.03643658 0.0006046618
## 119   119 0.03243929 0.2011529 0.02487736 0.0009071571 0.03624164 0.0006079265
## 120   120 0.03244283 0.2010514 0.02488226 0.0009134660 0.03636106 0.0006180777
## 121   121 0.03244427 0.2010609 0.02488148 0.0009148976 0.03652459 0.0006174848
## 122   122 0.03244945 0.2008182 0.02488680 0.0009119308 0.03631945 0.0006128012
## 123   123 0.03245581 0.2005432 0.02488725 0.0009123385 0.03640731 0.0006085068
## 124   124 0.03245297 0.2006619 0.02489173 0.0009129194 0.03626959 0.0006145295
## 125   125 0.03244814 0.2008762 0.02488508 0.0009165463 0.03660155 0.0006133011
## 126   126 0.03244762 0.2009118 0.02488784 0.0009098030 0.03625566 0.0006051267
## 127   127 0.03245295 0.2006458 0.02489099 0.0009066927 0.03576490 0.0006002143
## 128   128 0.03244573 0.2009272 0.02488850 0.0009060542 0.03555825 0.0005987504
## 129   129 0.03245275 0.2006177 0.02489327 0.0009063281 0.03553632 0.0006061074
## 130   130 0.03245175 0.2006523 0.02489165 0.0009027759 0.03513428 0.0005963687
## 131   131 0.03245967 0.2002988 0.02489617 0.0009018495 0.03494414 0.0005934144
## 132   132 0.03246058 0.2002667 0.02489823 0.0008957945 0.03452913 0.0005892603
## 133   133 0.03246303 0.2001717 0.02490650 0.0008976344 0.03470926 0.0005907284
## 134   134 0.03247011 0.1998487 0.02491807 0.0009019086 0.03478751 0.0005893136
## 135   135 0.03247108 0.1998252 0.02492261 0.0008933055 0.03467518 0.0005911620
## 136   136 0.03248096 0.1994335 0.02492854 0.0008891232 0.03476535 0.0005869376
## 137   137 0.03248134 0.1994475 0.02492800 0.0008962226 0.03526829 0.0005927458
## 138   138 0.03247980 0.1995227 0.02492928 0.0008980769 0.03532209 0.0005924291
## 139   139 0.03247748 0.1996114 0.02492956 0.0008924741 0.03527879 0.0005867558
## 140   140 0.03247990 0.1995765 0.02492891 0.0008990298 0.03574933 0.0005914160
## 141   141 0.03248075 0.1995527 0.02492423 0.0008933510 0.03537442 0.0005885740
## 142   142 0.03248481 0.1994200 0.02492960 0.0009004975 0.03571308 0.0006001831
## 143   143 0.03248921 0.1992345 0.02492911 0.0008977155 0.03575133 0.0005950214
## 144   144 0.03248899 0.1992627 0.02493004 0.0008993493 0.03599085 0.0005995837
## 145   145 0.03248802 0.1993090 0.02493222 0.0008996267 0.03602325 0.0005959545
## 146   146 0.03248786 0.1993068 0.02493270 0.0008944526 0.03596578 0.0005929660
## 147   147 0.03248599 0.1993795 0.02493548 0.0008916145 0.03579382 0.0005953827
## 148   148 0.03248715 0.1993376 0.02493134 0.0008931865 0.03584547 0.0005965928
## 149   149 0.03247967 0.1996832 0.02492525 0.0008901702 0.03580308 0.0005953164
## 150   150 0.03248030 0.1996398 0.02492509 0.0008893478 0.03575191 0.0005953159
## 151   151 0.03247908 0.1996842 0.02492283 0.0008847148 0.03555972 0.0005934208
## 152   152 0.03247867 0.1997285 0.02492220 0.0008871653 0.03549270 0.0005951561
## 153   153 0.03247685 0.1998173 0.02491872 0.0008855208 0.03560468 0.0005882683
## 154   154 0.03247364 0.1999607 0.02491683 0.0008817312 0.03550499 0.0005835038
## 155   155 0.03247643 0.1998412 0.02491757 0.0008767824 0.03537080 0.0005809154
## 156   156 0.03247929 0.1997479 0.02491496 0.0008802024 0.03546313 0.0005863423
## 157   157 0.03247793 0.1997868 0.02491210 0.0008852722 0.03547960 0.0005882372
## 158   158 0.03247792 0.1997528 0.02490856 0.0008765300 0.03516350 0.0005830026
## 159   159 0.03248270 0.1995561 0.02491019 0.0008805917 0.03509420 0.0005857595
## 160   160 0.03248318 0.1995455 0.02491316 0.0008814192 0.03502038 0.0005856051
## 161   161 0.03248355 0.1995108 0.02491171 0.0008794684 0.03483803 0.0005797311
## 162   162 0.03248525 0.1994376 0.02491218 0.0008746398 0.03473257 0.0005787146
## 163   163 0.03248621 0.1994077 0.02491516 0.0008739026 0.03462654 0.0005826598
## 164   164 0.03248924 0.1992807 0.02491824 0.0008720160 0.03443113 0.0005792588
## 165   165 0.03249167 0.1991579 0.02492203 0.0008705328 0.03431573 0.0005766801
## 166   166 0.03248994 0.1992379 0.02492063 0.0008728192 0.03446324 0.0005742793
## 167   167 0.03248709 0.1993745 0.02491879 0.0008738200 0.03460814 0.0005754590
## 168   168 0.03248919 0.1993036 0.02491820 0.0008838534 0.03487807 0.0005838077
## 169   169 0.03248811 0.1993721 0.02491719 0.0008843275 0.03508555 0.0005833348
## 170   170 0.03248662 0.1994363 0.02491756 0.0008838456 0.03500095 0.0005835964
## 171   171 0.03248482 0.1995003 0.02491440 0.0008803264 0.03487570 0.0005846282
## 172   172 0.03248587 0.1994747 0.02491488 0.0008788198 0.03482278 0.0005840143
## 173   173 0.03248417 0.1995409 0.02491196 0.0008764807 0.03481496 0.0005801069
## 174   174 0.03248589 0.1994738 0.02491325 0.0008741659 0.03477782 0.0005798526
## 175   175 0.03248277 0.1996001 0.02491032 0.0008750532 0.03475663 0.0005782106
## 176   176 0.03248368 0.1995793 0.02491132 0.0008694755 0.03468218 0.0005744036
## 177   177 0.03248524 0.1995088 0.02491310 0.0008680756 0.03465088 0.0005747126
## 178   178 0.03248514 0.1995054 0.02491196 0.0008673279 0.03445212 0.0005773924
## 179   179 0.03248757 0.1994074 0.02491225 0.0008679335 0.03445758 0.0005740555
## 180   180 0.03248661 0.1994323 0.02491405 0.0008659456 0.03450038 0.0005732458
## 181   181 0.03248540 0.1994917 0.02491359 0.0008671068 0.03457182 0.0005745833
## 182   182 0.03248598 0.1994578 0.02491708 0.0008678656 0.03450642 0.0005771587
## 183   183 0.03248375 0.1995698 0.02491396 0.0008716514 0.03458035 0.0005795246
## 184   184 0.03248119 0.1996894 0.02491178 0.0008759805 0.03463431 0.0005797688
## 185   185 0.03248638 0.1994500 0.02491678 0.0008754796 0.03452164 0.0005769543
## 186   186 0.03248448 0.1995271 0.02491464 0.0008747700 0.03450348 0.0005784177
## 187   187 0.03248309 0.1995821 0.02491304 0.0008750297 0.03448864 0.0005776661
## 188   188 0.03248527 0.1994988 0.02491386 0.0008785071 0.03462768 0.0005825981
## 189   189 0.03248519 0.1994901 0.02491437 0.0008808105 0.03462460 0.0005838096
## 190   190 0.03248348 0.1995678 0.02491185 0.0008812150 0.03465522 0.0005834497
## 191   191 0.03248582 0.1994765 0.02491456 0.0008808687 0.03470206 0.0005834786
## 192   192 0.03248879 0.1993554 0.02491624 0.0008827440 0.03480037 0.0005832946
## 193   193 0.03248836 0.1993937 0.02491681 0.0008827079 0.03491264 0.0005871685
## 194   194 0.03249063 0.1993061 0.02491628 0.0008808992 0.03496958 0.0005889331
## 195   195 0.03249181 0.1992811 0.02491624 0.0008805230 0.03511208 0.0005900521
## 196   196 0.03249411 0.1991847 0.02491980 0.0008784703 0.03503171 0.0005888367
## 197   197 0.03249717 0.1990595 0.02492232 0.0008766760 0.03488136 0.0005876382
## 198   198 0.03249700 0.1990701 0.02492205 0.0008757393 0.03486623 0.0005874632
## 199   199 0.03249587 0.1991170 0.02492070 0.0008769262 0.03482918 0.0005868592
## 200   200 0.03249736 0.1990576 0.02492260 0.0008793085 0.03491794 0.0005910458
## 201   201 0.03249919 0.1989773 0.02492408 0.0008772338 0.03477848 0.0005881223
## 202   202 0.03249803 0.1990354 0.02492349 0.0008782833 0.03478624 0.0005909289
## 203   203 0.03249677 0.1990836 0.02492243 0.0008757856 0.03467841 0.0005860012
## 204   204 0.03249710 0.1990668 0.02492457 0.0008767562 0.03466324 0.0005860671
## 205   205 0.03249818 0.1990258 0.02492668 0.0008780947 0.03469432 0.0005871516
## 206   206 0.03249791 0.1990267 0.02492721 0.0008773839 0.03471409 0.0005887559
## 207   207 0.03249862 0.1990058 0.02492790 0.0008775362 0.03474592 0.0005908043
## 208   208 0.03249686 0.1990829 0.02492739 0.0008758180 0.03471911 0.0005886337
## 209   209 0.03249806 0.1990351 0.02492811 0.0008743648 0.03465560 0.0005880437
## 210   210 0.03249678 0.1990961 0.02492827 0.0008739620 0.03463195 0.0005884302
## 211   211 0.03249636 0.1991085 0.02492883 0.0008749114 0.03466904 0.0005890931
## 212   212 0.03249731 0.1990641 0.02492845 0.0008751065 0.03463876 0.0005900654
## 213   213 0.03249735 0.1990601 0.02492847 0.0008754028 0.03463836 0.0005904566
## 214   214 0.03249683 0.1990798 0.02492931 0.0008746378 0.03454403 0.0005900086
## 215   215 0.03249741 0.1990594 0.02492879 0.0008751510 0.03453079 0.0005909715
## 216   216 0.03249838 0.1990236 0.02492900 0.0008754930 0.03451170 0.0005912768
## 217   217 0.03249811 0.1990289 0.02492790 0.0008753197 0.03447562 0.0005900866
## 218   218 0.03249877 0.1990066 0.02492862 0.0008757781 0.03449711 0.0005906042
## 219   219 0.03249911 0.1989887 0.02492877 0.0008756685 0.03447755 0.0005902879
## 220   220 0.03249969 0.1989647 0.02492930 0.0008755146 0.03444921 0.0005898006
## 221   221 0.03249966 0.1989690 0.02492888 0.0008759477 0.03447460 0.0005896631
## 222   222 0.03249987 0.1989670 0.02492902 0.0008754698 0.03450162 0.0005897607
## 223   223 0.03249982 0.1989739 0.02492932 0.0008756809 0.03452805 0.0005894229
## 224   224 0.03250048 0.1989442 0.02492961 0.0008750176 0.03448473 0.0005889142
## 225   225 0.03250152 0.1989007 0.02492986 0.0008742688 0.03447501 0.0005895298
## 226   226 0.03250154 0.1988985 0.02493019 0.0008751578 0.03452779 0.0005899196
## 227   227 0.03250120 0.1989146 0.02493008 0.0008746615 0.03453660 0.0005894590
## 228   228 0.03250119 0.1989153 0.02493016 0.0008744449 0.03450364 0.0005893642
## 229   229 0.03250118 0.1989170 0.02493060 0.0008749421 0.03451212 0.0005898776
## 230   230 0.03250202 0.1988837 0.02493127 0.0008753845 0.03451525 0.0005907059
## 231   231 0.03250174 0.1988955 0.02493155 0.0008752620 0.03453263 0.0005906768
## 232   232 0.03250172 0.1988959 0.02493129 0.0008753881 0.03455645 0.0005909734
## 233   233 0.03250199 0.1988857 0.02493163 0.0008752934 0.03453992 0.0005910003
## 234   234 0.03250185 0.1988908 0.02493129 0.0008752629 0.03453640 0.0005907534
## 235   235 0.03250217 0.1988765 0.02493170 0.0008753765 0.03453248 0.0005912316
## 236   236 0.03250231 0.1988701 0.02493181 0.0008753587 0.03452230 0.0005914375
## 237   237 0.03250231 0.1988711 0.02493180 0.0008753858 0.03452655 0.0005915679
## 238   238 0.03250246 0.1988655 0.02493188 0.0008753904 0.03452419 0.0005916808
## 239   239 0.03250247 0.1988651 0.02493187 0.0008753765 0.03452171 0.0005917486
## 240   240 0.03250253 0.1988622 0.02493193 0.0008752879 0.03451534 0.0005917198
## [1] "Best Model"
##   nvmax
## 7     7

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.010989e+00  2.005707e+00  2.016270e+00
## x4          -4.399973e-05 -6.159867e-05 -2.640079e-05
## x7           1.108087e-02  9.833384e-03  1.232835e-02
## x9           3.310416e-03  2.662524e-03  3.958309e-03
## x17          1.507310e-03  8.723715e-04  2.142248e-03
## stat98       3.548406e-03  3.072152e-03  4.024661e-03
## stat110     -3.282208e-03 -3.767312e-03 -2.797105e-03
## x18.sqrt     2.527794e-02  2.342698e-02  2.712889e-02

Test

if (algo.backward.caret == TRUE){
  test.model(model.backward, data.test
             ,method = 'leapBackward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}

## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.044   2.084   2.097   2.097   2.109   2.147 
## [1] "leapBackward  Test MSE: 0.000942359887602509"

Stepwise Selection with CV

Train

if (algo.stepwise.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapSeq"
                                   ,feature.names = feature.names)
  model.stepwise = returned$model
  id = returned$id
}

## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 15 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.03401434 0.1129284 0.02643588 0.001653505 0.02622841 0.0011028596
## 2       2 0.03327509 0.1508562 0.02584895 0.001824017 0.02882595 0.0012983084
## 3       3 0.03268776 0.1806362 0.02522895 0.001724857 0.02926289 0.0012428418
## 4       4 0.03221824 0.2037167 0.02457710 0.001619741 0.02635159 0.0011552055
## 5       5 0.03198566 0.2150295 0.02440800 0.001623696 0.02619591 0.0011599993
## 6       6 0.03196348 0.2163788 0.02441940 0.001696140 0.02841727 0.0011742856
## 7       7 0.03180550 0.2239996 0.02428267 0.001715946 0.03089328 0.0011996369
## 8       8 0.03174826 0.2265913 0.02425835 0.001683503 0.02873522 0.0011714720
## 9       9 0.03178904 0.2246853 0.02428039 0.001704478 0.02955390 0.0011706832
## 10     10 0.03177374 0.2253224 0.02427874 0.001691829 0.02774504 0.0011286754
## 11     11 0.03177040 0.2255159 0.02427981 0.001670797 0.02729749 0.0011237625
## 12     12 0.03176208 0.2259251 0.02429125 0.001680949 0.02809431 0.0011348954
## 13     13 0.03175935 0.2260897 0.02428991 0.001688991 0.02737024 0.0011459004
## 14     14 0.03174727 0.2267631 0.02428874 0.001676773 0.02679936 0.0011309569
## 15     15 0.03172887 0.2277161 0.02427692 0.001696985 0.02798228 0.0011500434
## 16     16 0.03175348 0.2266272 0.02428481 0.001702021 0.02818457 0.0011457179
## 17     17 0.03206444 0.2109655 0.02451358 0.002194672 0.06313721 0.0014994882
## 18     18 0.03175003 0.2267465 0.02428602 0.001665137 0.02777001 0.0011004389
## 19     19 0.03214303 0.2055958 0.02461990 0.001317680 0.05425794 0.0008439660
## 20     20 0.03177849 0.2253463 0.02431685 0.001646734 0.02577676 0.0010747508
## 21     21 0.03179040 0.2247855 0.02433509 0.001665307 0.02577533 0.0010965654
## 22     22 0.03180574 0.2241727 0.02434164 0.001696696 0.02671741 0.0011302383
## 23     23 0.03181246 0.2239205 0.02434062 0.001684177 0.02648960 0.0011146907
## 24     24 0.03184966 0.2221161 0.02436100 0.001659953 0.02556547 0.0011108724
## 25     25 0.03187019 0.2211773 0.02436632 0.001669196 0.02587348 0.0011205983
## 26     26 0.03250381 0.1888801 0.02495444 0.002244775 0.07303781 0.0017187798
## 27     27 0.03211741 0.2074997 0.02456334 0.001483839 0.04880065 0.0011074442
## 28     28 0.03189075 0.2202171 0.02438434 0.001635757 0.02421663 0.0010901348
## 29     29 0.03213757 0.2065274 0.02459099 0.001463685 0.04797825 0.0010997221
## 30     30 0.03249771 0.1898242 0.02482952 0.002418352 0.07401254 0.0017226152
## 31     31 0.03228152 0.1995812 0.02470678 0.001302369 0.04767336 0.0008573188
## 32     32 0.03258683 0.1861198 0.02490156 0.002647267 0.07643979 0.0018825450
## 33     33 0.03195319 0.2174742 0.02442165 0.001647238 0.02376143 0.0011078370
## 34     34 0.03262077 0.1836188 0.02503088 0.002241282 0.07126069 0.0017274456
## 35     35 0.03199660 0.2155346 0.02446352 0.001655760 0.02410613 0.0011286020
## 36     36 0.03228777 0.2004763 0.02469530 0.001684752 0.04550704 0.0012560910
## 37     37 0.03331477 0.1484894 0.02556535 0.002269138 0.07860681 0.0015806007
## 38     38 0.03291903 0.1678050 0.02517469 0.002031300 0.07435351 0.0014021506
## 39     39 0.03204275 0.2135917 0.02449645 0.001690206 0.02585637 0.0011420426
## 40     40 0.03296737 0.1653948 0.02524930 0.001623385 0.06464516 0.0010728143
## 41     41 0.03241825 0.1936417 0.02480632 0.001291442 0.04693351 0.0008045908
## 42     42 0.03239022 0.1958657 0.02482728 0.001844065 0.05527604 0.0012723186
## 43     43 0.03273787 0.1771919 0.02513165 0.001444838 0.06274835 0.0008985786
## 44     44 0.03323699 0.1528356 0.02544255 0.002547447 0.08026996 0.0019248498
## 45     45 0.03235273 0.1980189 0.02472547 0.002075100 0.05724407 0.0014270292
## 46     46 0.03300420 0.1649808 0.02528495 0.002429234 0.07611030 0.0019454112
## 47     47 0.03211235 0.2106840 0.02455083 0.001669872 0.02623471 0.0011451270
## 48     48 0.03263821 0.1822700 0.02506306 0.001649335 0.06448309 0.0012482264
## 49     49 0.03213880 0.2095008 0.02457078 0.001667150 0.02598830 0.0011444590
## 50     50 0.03246475 0.1937045 0.02483119 0.002284708 0.05683085 0.0016416954
## 51     51 0.03250667 0.1899744 0.02489496 0.001274678 0.04499547 0.0007997801
## 52     52 0.03251844 0.1907765 0.02488882 0.002139593 0.05601955 0.0016825508
## 53     53 0.03218520 0.2074853 0.02460408 0.001680245 0.02569057 0.0011566429
## 54     54 0.03248482 0.1918802 0.02483764 0.001696982 0.04576401 0.0012992711
## 55     55 0.03244137 0.1942300 0.02481074 0.002080333 0.05592574 0.0014561591
## 56     56 0.03247961 0.1927773 0.02485369 0.002063278 0.05303584 0.0014017893
## 57     57 0.03282353 0.1719006 0.02516600 0.001951148 0.07544187 0.0014741796
## 58     58 0.03245530 0.1939349 0.02483357 0.002088400 0.05350510 0.0014171075
## 59     59 0.03245929 0.1939068 0.02480721 0.002032351 0.05203691 0.0014522380
## 60     60 0.03279543 0.1771452 0.02508105 0.002381620 0.06860525 0.0018499568
## 61     61 0.03281689 0.1761641 0.02514179 0.002400831 0.06853259 0.0017976515
## 62     62 0.03249214 0.1925243 0.02482792 0.002026997 0.05182746 0.0014422530
## 63     63 0.03272874 0.1791926 0.02511096 0.002180183 0.06838249 0.0017070295
## 64     64 0.03306454 0.1622283 0.02534545 0.002050879 0.06837558 0.0016123300
## 65     65 0.03280207 0.1763863 0.02516193 0.002147267 0.06743471 0.0014775109
## 66     66 0.03269351 0.1807722 0.02505395 0.002121355 0.06763452 0.0014955804
## 67     67 0.03226483 0.2044421 0.02468764 0.001668494 0.02341296 0.0011662386
## 68     68 0.03226478 0.2044286 0.02468558 0.001666144 0.02389298 0.0011652132
## 69     69 0.03262107 0.1854286 0.02498466 0.001249269 0.04348572 0.0007963032
## 70     70 0.03255918 0.1890959 0.02497699 0.001809519 0.05168830 0.0012859506
## 71     71 0.03261032 0.1859698 0.02499245 0.001241030 0.04321487 0.0007949536
## 72     72 0.03321139 0.1577092 0.02540939 0.002755707 0.07796841 0.0021033744
## 73     73 0.03298394 0.1656754 0.02532317 0.002041014 0.07129820 0.0014878511
## 74     74 0.03228327 0.2037517 0.02471874 0.001638846 0.02334875 0.0011384769
## 75     75 0.03295364 0.1695294 0.02526005 0.001928746 0.06154688 0.0013208723
## 76     76 0.03271973 0.1799920 0.02508454 0.002088526 0.06577952 0.0014351186
## 77     77 0.03250844 0.1908755 0.02490986 0.001449528 0.04485167 0.0011223007
## 78     78 0.03295746 0.1707017 0.02524681 0.002565217 0.06985802 0.0019467604
## 79     79 0.03257449 0.1886397 0.02500205 0.001776260 0.05095803 0.0012417906
## 80     80 0.03316411 0.1575984 0.02548874 0.001708500 0.06749191 0.0010147627
## 81     81 0.03307350 0.1619718 0.02542449 0.002221264 0.07734206 0.0017000431
## 82     82 0.03229061 0.2036087 0.02471179 0.001636826 0.02330872 0.0011311823
## 83     83 0.03262632 0.1868525 0.02498852 0.002095519 0.05479223 0.0016271073
## 84     84 0.03275192 0.1787173 0.02507156 0.002085890 0.06736939 0.0014994144
## 85     85 0.03321907 0.1549295 0.02550772 0.001647625 0.06428258 0.0013549780
## 86     86 0.03328198 0.1508014 0.02551776 0.002402469 0.08281620 0.0017389507
## 87     87 0.03228488 0.2039189 0.02473562 0.001630830 0.02335386 0.0011034536
## 88     88 0.03257383 0.1892871 0.02494192 0.002060085 0.05477581 0.0013741006
## 89     89 0.03257649 0.1892073 0.02494040 0.002066591 0.05489752 0.0013852654
## 90     90 0.03283803 0.1750762 0.02519661 0.002002383 0.06213163 0.0014341643
## 91     91 0.03229698 0.2034541 0.02474503 0.001632224 0.02324343 0.0011049377
## 92     92 0.03258301 0.1891117 0.02493364 0.002027120 0.05287455 0.0014228220
## 93     93 0.03304903 0.1638190 0.02531375 0.002140052 0.07220758 0.0015435360
## 94     94 0.03246428 0.1933323 0.02488758 0.001774365 0.05295594 0.0012456497
## 95     95 0.03259284 0.1884901 0.02498461 0.002087789 0.05412110 0.0016161376
## 96     96 0.03261258 0.1882501 0.02499424 0.002298767 0.05699648 0.0016630436
## 97     97 0.03300158 0.1652211 0.02532934 0.002057178 0.07337230 0.0014809526
## 98     98 0.03345643 0.1445703 0.02567312 0.002617120 0.08351665 0.0019331891
## 99     99 0.03294791 0.1671876 0.02529478 0.001888154 0.07322293 0.0013598843
## 100   100 0.03287262 0.1738470 0.02518035 0.002032279 0.06506466 0.0015643397
## 101   101 0.03228132 0.2042434 0.02472874 0.001627242 0.02336352 0.0011082874
## 102   102 0.03260369 0.1868793 0.02500647 0.001242104 0.04328665 0.0007549912
## 103   103 0.03291016 0.1729986 0.02521414 0.002397935 0.07003013 0.0018602113
## 104   104 0.03279125 0.1759676 0.02517462 0.001370708 0.06085255 0.0008719858
## 105   105 0.03328186 0.1537817 0.02556679 0.002263277 0.07108730 0.0017230143
## 106   106 0.03291344 0.1729001 0.02522661 0.002404325 0.07016072 0.0018658966
## 107   107 0.03262997 0.1876371 0.02501447 0.002294528 0.05626876 0.0016465073
## 108   108 0.03314918 0.1596499 0.02546207 0.002327050 0.07378182 0.0018043083
## 109   109 0.03257385 0.1895828 0.02492604 0.002059456 0.05458835 0.0013739372
## 110   110 0.03289364 0.1735112 0.02524204 0.002148245 0.06789440 0.0015205738
## 111   111 0.03305882 0.1624658 0.02539247 0.001753960 0.07186899 0.0010745397
## 112   112 0.03229835 0.2037367 0.02473685 0.001635951 0.02400001 0.0010923628
## 113   113 0.03229482 0.2038396 0.02474034 0.001638344 0.02376451 0.0010968095
## 114   114 0.03261506 0.1882042 0.02496552 0.002063896 0.05490034 0.0014634566
## 115   115 0.03361601 0.1343346 0.02589038 0.002371414 0.08525429 0.0016541353
## 116   116 0.03264483 0.1865639 0.02503704 0.002096711 0.05498877 0.0016284240
## 117   117 0.03261236 0.1883247 0.02496670 0.002061670 0.05521925 0.0014575536
## 118   118 0.03338036 0.1457032 0.02564948 0.001654605 0.07587359 0.0011081997
## 119   119 0.03262629 0.1878670 0.02497760 0.002075768 0.05576566 0.0014674712
## 120   120 0.03252391 0.1919071 0.02491340 0.001630496 0.03957787 0.0011540329
## 121   121 0.03309752 0.1640472 0.02542284 0.002234129 0.06461930 0.0016795592
## 122   122 0.03258217 0.1895586 0.02500983 0.001745275 0.04640000 0.0011454174
## 123   123 0.03259946 0.1891478 0.02500495 0.001977500 0.04602470 0.0015128234
## 124   124 0.03300440 0.1687039 0.02529514 0.002417946 0.06890697 0.0016662179
## 125   125 0.03257350 0.1895931 0.02496026 0.001476185 0.04834021 0.0011169360
## 126   126 0.03254727 0.1915164 0.02492668 0.001974675 0.04855876 0.0013310641
## 127   127 0.03232153 0.2029851 0.02475569 0.001652642 0.02520058 0.0010948364
## 128   128 0.03257433 0.1895487 0.02496351 0.001473335 0.04817835 0.0011213601
## 129   129 0.03289050 0.1737329 0.02523958 0.001608950 0.04829278 0.0012136395
## 130   130 0.03306900 0.1649650 0.02539991 0.002071889 0.05976104 0.0014810799
## 131   131 0.03233185 0.2025632 0.02475994 0.001645547 0.02538642 0.0010918386
## 132   132 0.03267870 0.1842408 0.02502697 0.001823499 0.04608019 0.0012717573
## 133   133 0.03233836 0.2022724 0.02476608 0.001650099 0.02566741 0.0010898807
## 134   134 0.03234108 0.2021869 0.02476832 0.001656163 0.02589855 0.0010954175
## 135   135 0.03279062 0.1799568 0.02511432 0.001754052 0.05521837 0.0013228049
## 136   136 0.03235807 0.2014833 0.02478327 0.001661997 0.02586826 0.0011045347
## 137   137 0.03255512 0.1914497 0.02494978 0.001868273 0.03779094 0.0013625658
## 138   138 0.03306970 0.1642444 0.02534843 0.001755331 0.05773348 0.0012740594
## 139   139 0.03249571 0.1941728 0.02488761 0.001857086 0.03942406 0.0011920233
## 140   140 0.03236155 0.2013668 0.02478608 0.001673968 0.02633410 0.0011112685
## 141   141 0.03284852 0.1741064 0.02518579 0.001503225 0.04852244 0.0009126194
## 142   142 0.03236156 0.2013947 0.02478221 0.001664649 0.02627363 0.0011005369
## 143   143 0.03236602 0.2012424 0.02478322 0.001672712 0.02645275 0.0011020062
## 144   144 0.03278432 0.1806095 0.02510076 0.002095259 0.05002206 0.0015384312
## 145   145 0.03237377 0.2008984 0.02479473 0.001661227 0.02618151 0.0010970704
## 146   146 0.03237690 0.2007521 0.02480046 0.001658297 0.02588431 0.0010958361
## 147   147 0.03252594 0.1928273 0.02491840 0.001861260 0.04053640 0.0012375127
## 148   148 0.03246477 0.1946532 0.02489396 0.001709253 0.04085018 0.0011508970
## 149   149 0.03285264 0.1745578 0.02520337 0.001853581 0.05136690 0.0014273827
## 150   150 0.03260689 0.1891565 0.02499681 0.001501988 0.04480277 0.0011278006
## 151   151 0.03278620 0.1801488 0.02515961 0.001971862 0.04550450 0.0014935517
## 152   152 0.03237819 0.2007557 0.02479054 0.001669073 0.02664179 0.0011072442
## 153   153 0.03286103 0.1771845 0.02516624 0.001999431 0.05582568 0.0013450261
## 154   154 0.03238298 0.2005499 0.02478946 0.001667611 0.02654405 0.0011069955
## 155   155 0.03260168 0.1899737 0.02500417 0.002014337 0.03998462 0.0014603286
## 156   156 0.03260598 0.1897980 0.02500760 0.002012688 0.03990963 0.0014594781
## 157   157 0.03261783 0.1894205 0.02495619 0.001956190 0.04613639 0.0013713406
## 158   158 0.03261531 0.1893939 0.02501396 0.002025781 0.04058825 0.0014715112
## 159   159 0.03239045 0.2002100 0.02479280 0.001657961 0.02611324 0.0011044963
## 160   160 0.03248830 0.1936791 0.02489028 0.001708234 0.04169251 0.0011636675
## 161   161 0.03239625 0.1999569 0.02479616 0.001662452 0.02606409 0.0011128582
## 162   162 0.03292526 0.1722275 0.02527468 0.002204582 0.05454406 0.0016736126
## 163   163 0.03239916 0.1998226 0.02479641 0.001658185 0.02616526 0.0011067244
## 164   164 0.03274750 0.1814160 0.02507704 0.001819527 0.04764959 0.0013012064
## 165   165 0.03239918 0.1998215 0.02479967 0.001654886 0.02613508 0.0011034698
## 166   166 0.03278017 0.1802718 0.02511778 0.001705242 0.05284472 0.0011876572
## 167   167 0.03261092 0.1888169 0.02497650 0.001648381 0.03919726 0.0012063746
## 168   168 0.03240307 0.1996735 0.02479781 0.001660347 0.02614156 0.0011053862
## 169   169 0.03305550 0.1673540 0.02531600 0.002083427 0.05533826 0.0016334200
## 170   170 0.03291456 0.1722155 0.02525174 0.001946777 0.06017784 0.0012768254
## 171   171 0.03240844 0.1994291 0.02480448 0.001659622 0.02597122 0.0011098408
## 172   172 0.03264960 0.1882103 0.02498493 0.001979779 0.04712272 0.0014081457
## 173   173 0.03250507 0.1930446 0.02490607 0.001711327 0.04146185 0.0011677271
## 174   174 0.03241102 0.1993065 0.02480963 0.001664697 0.02606782 0.0011161495
## 175   175 0.03240922 0.1993815 0.02480860 0.001662166 0.02590927 0.0011130963
## 176   176 0.03307120 0.1667639 0.02533381 0.002102221 0.05618104 0.0016524572
## 177   177 0.03241017 0.1993653 0.02481099 0.001667773 0.02608223 0.0011154671
## 178   178 0.03241202 0.1992881 0.02481172 0.001672370 0.02627175 0.0011187910
## 179   179 0.03241165 0.1992964 0.02481252 0.001668491 0.02606235 0.0011147208
## 180   180 0.03269228 0.1848031 0.02501932 0.001256724 0.03301180 0.0007675867
## 181   181 0.03240932 0.1994042 0.02481230 0.001665697 0.02604145 0.0011110781
## 182   182 0.03269146 0.1848576 0.02501934 0.001256030 0.03299519 0.0007662943
## 183   183 0.03241353 0.1992355 0.02481809 0.001667409 0.02596555 0.0011126169
## 184   184 0.03257661 0.1907354 0.02494694 0.001889113 0.04214910 0.0012670350
## 185   185 0.03267071 0.1844930 0.02504554 0.001924944 0.05173424 0.0013053749
## 186   186 0.03287278 0.1764686 0.02521308 0.001724943 0.05186801 0.0013896001
## 187   187 0.03262875 0.1884915 0.02498907 0.001906146 0.03978826 0.0014003773
## 188   188 0.03322402 0.1571399 0.02550510 0.001836998 0.05043989 0.0014003086
## 189   189 0.03241555 0.1991533 0.02482205 0.001668537 0.02592897 0.0011155080
## 190   190 0.03267269 0.1845255 0.02503786 0.001930727 0.05131710 0.0012492323
## 191   191 0.03241599 0.1991400 0.02482068 0.001668057 0.02596483 0.0011141172
## 192   192 0.03267492 0.1873278 0.02500848 0.002016569 0.04858424 0.0014293491
## 193   193 0.03241231 0.1992892 0.02481827 0.001667457 0.02602182 0.0011158739
## 194   194 0.03269444 0.1847306 0.02502104 0.001254875 0.03317374 0.0007727593
## 195   195 0.03241282 0.1992572 0.02482012 0.001667109 0.02598829 0.0011152746
## 196   196 0.03241304 0.1992417 0.02482241 0.001666622 0.02595122 0.0011157063
## 197   197 0.03241304 0.1992360 0.02482040 0.001663900 0.02579357 0.0011128488
## 198   198 0.03267826 0.1871664 0.02501633 0.002019834 0.04876076 0.0014420739
## 199   199 0.03291366 0.1759317 0.02524279 0.002317305 0.05593881 0.0017293642
## 200   200 0.03258573 0.1904632 0.02494236 0.001896526 0.04139322 0.0012050762
## 201   201 0.03270438 0.1852210 0.02507770 0.001789592 0.04722276 0.0011755736
## 202   202 0.03262876 0.1885284 0.02499337 0.001896738 0.03903597 0.0013959521
## 203   203 0.03242174 0.1988564 0.02482805 0.001665860 0.02569423 0.0011156000
## 204   204 0.03242247 0.1988211 0.02482887 0.001666421 0.02570974 0.0011175675
## 205   205 0.03242220 0.1988275 0.02482759 0.001664984 0.02569562 0.0011165821
## 206   206 0.03265479 0.1876850 0.02505579 0.002045807 0.04121854 0.0015022222
## 207   207 0.03242159 0.1988437 0.02482838 0.001663210 0.02562531 0.0011173476
## 208   208 0.03242107 0.1988813 0.02482874 0.001662465 0.02566752 0.0011181276
## 209   209 0.03265413 0.1877302 0.02505404 0.002045260 0.04119437 0.0014993132
## 210   210 0.03242222 0.1988449 0.02482784 0.001664957 0.02574449 0.0011182482
## 211   211 0.03265668 0.1876433 0.02505607 0.002051778 0.04149237 0.0015042564
## 212   212 0.03241903 0.1989840 0.02482523 0.001663797 0.02582182 0.0011166440
## 213   213 0.03251013 0.1929782 0.02491608 0.001708898 0.04040613 0.0011654902
## 214   214 0.03270688 0.1842858 0.02503617 0.001241902 0.03332938 0.0007641282
## 215   215 0.03241820 0.1990202 0.02482473 0.001662506 0.02575708 0.0011178729
## 216   216 0.03265498 0.1877258 0.02504783 0.002055481 0.04177207 0.0014952481
## 217   217 0.03241672 0.1990814 0.02482366 0.001661302 0.02571403 0.0011189592
## 218   218 0.03259150 0.1903139 0.02494966 0.001910639 0.04217300 0.0012201240
## 219   219 0.03241515 0.1991480 0.02482193 0.001662119 0.02569389 0.0011199403
## 220   220 0.03267727 0.1867323 0.02507107 0.001492977 0.04605700 0.0011700818
## 221   221 0.03273181 0.1820802 0.02509055 0.001939639 0.04990527 0.0014310582
## 222   222 0.03259741 0.1899927 0.02496633 0.001908312 0.04301046 0.0012872989
## 223   223 0.03260732 0.1896895 0.02496608 0.001937122 0.04363399 0.0012374189
## 224   224 0.03252101 0.1925506 0.02493419 0.001717379 0.04174000 0.0011835719
## 225   225 0.03271167 0.1831596 0.02507319 0.001974520 0.05334456 0.0012816496
## 226   226 0.03241783 0.1990293 0.02482596 0.001661740 0.02567120 0.0011194123
## 227   227 0.03268637 0.1869863 0.02501947 0.002031423 0.04915889 0.0014510991
## 228   228 0.03273407 0.1820114 0.02509692 0.001942308 0.04993416 0.0014398433
## 229   229 0.03264507 0.1876066 0.02502042 0.001673915 0.04129320 0.0012549416
## 230   230 0.03241756 0.1990477 0.02482596 0.001663162 0.02578357 0.0011204303
## 231   231 0.03241816 0.1990219 0.02482649 0.001663076 0.02576580 0.0011205362
## 232   232 0.03294414 0.1720876 0.02529141 0.002277427 0.05900339 0.0015830135
## 233   233 0.03269721 0.1866651 0.02502766 0.002052690 0.05008129 0.0014681652
## 234   234 0.03306620 0.1675635 0.02535667 0.002410782 0.05659359 0.0017432456
## 235   235 0.03241782 0.1990370 0.02482623 0.001662943 0.02578869 0.0011204801
## 236   236 0.03282883 0.1787823 0.02513675 0.002126672 0.05168182 0.0015219176
## 237   237 0.03339563 0.1522882 0.02570700 0.002077293 0.06504305 0.0015201503
## 238   238 0.03297163 0.1709288 0.02531410 0.001783632 0.06197715 0.0012637892
## 239   239 0.03363603 0.1404458 0.02591614 0.001965316 0.06426498 0.0015099006
## 240   240 0.03241747 0.1990510 0.02482615 0.001663024 0.02579431 0.0011205616
## [1] "Best Model"
##    nvmax
## 15    15

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.967507e+00  1.951372e+00  1.983643e+00
## x4          -5.029613e-05 -6.769517e-05 -3.289709e-05
## x7           1.044485e-02  9.210735e-03  1.167897e-02
## x8           4.600676e-04  1.745882e-04  7.455470e-04
## x9           3.012246e-03  2.373171e-03  3.651320e-03
## x10          1.124834e-03  5.308712e-04  1.718798e-03
## x11          2.144872e+05  7.225403e+04  3.567204e+05
## x16          1.235851e-03  8.222581e-04  1.649445e-03
## x17          1.500100e-03  8.677607e-04  2.132439e-03
## x21          1.289568e-04  4.739125e-05  2.105223e-04
## stat14      -8.098545e-04 -1.285739e-03 -3.339704e-04
## stat22      -6.216928e-04 -1.103225e-03 -1.401604e-04
## stat98       3.592799e-03  3.120309e-03  4.065290e-03
## stat110     -3.234551e-03 -3.714185e-03 -2.754917e-03
## stat149     -6.442636e-04 -1.125085e-03 -1.634420e-04
## x18.sqrt     2.633233e-02  2.451034e-02  2.815431e-02

Test

if (algo.stepwise.caret == TRUE){
  test.model(model.stepwise, data.test
             ,method = 'leapSeq',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
  
}

## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.033   2.085   2.097   2.097   2.109   2.151 
## [1] "leapSeq  Test MSE: 0.000918627674071896"

LASSO with CV

Train

if (algo.LASSO.caret == TRUE){
  set.seed(1)
  tune.grid= expand.grid(alpha = 1,lambda = 10^seq(from=-4,to=-2,length=100))
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "glmnet"
                                   ,subopt = 'LASSO'
                                   ,tune.grid = tune.grid
                                   ,feature.names = feature.names)
  model.LASSO.caret = returned$model
}

## Aggregating results
## Selecting tuning parameters
## Fitting alpha = 1, lambda = 0.000559 on full training set
## glmnet 
## 
## 5584 samples
##  240 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   lambda        RMSE        Rsquared   MAE       
##   0.0001000000  0.03228576  0.2064380  0.02476806
##   0.0001047616  0.03227771  0.2067390  0.02476232
##   0.0001097499  0.03226954  0.2070462  0.02475657
##   0.0001149757  0.03226119  0.2073622  0.02475076
##   0.0001204504  0.03225267  0.2076868  0.02474504
##   0.0001261857  0.03224387  0.2080246  0.02473914
##   0.0001321941  0.03223491  0.2083699  0.02473328
##   0.0001384886  0.03222580  0.2087234  0.02472756
##   0.0001450829  0.03221656  0.2090844  0.02472199
##   0.0001519911  0.03220742  0.2094424  0.02471646
##   0.0001592283  0.03219816  0.2098075  0.02471083
##   0.0001668101  0.03218869  0.2101859  0.02470498
##   0.0001747528  0.03217910  0.2105728  0.02469923
##   0.0001830738  0.03216926  0.2109742  0.02469306
##   0.0001917910  0.03215933  0.2113839  0.02468686
##   0.0002009233  0.03214933  0.2118013  0.02468058
##   0.0002104904  0.03213924  0.2122268  0.02467435
##   0.0002205131  0.03212904  0.2126618  0.02466789
##   0.0002310130  0.03211883  0.2131024  0.02466143
##   0.0002420128  0.03210841  0.2135598  0.02465457
##   0.0002535364  0.03209829  0.2140101  0.02464786
##   0.0002656088  0.03208856  0.2144518  0.02464084
##   0.0002782559  0.03207945  0.2148724  0.02463401
##   0.0002915053  0.03207058  0.2152909  0.02462730
##   0.0003053856  0.03206206  0.2157013  0.02462064
##   0.0003199267  0.03205339  0.2161297  0.02461439
##   0.0003351603  0.03204509  0.2165493  0.02460858
##   0.0003511192  0.03203737  0.2169497  0.02460353
##   0.0003678380  0.03203043  0.2173232  0.02459932
##   0.0003853529  0.03202361  0.2176976  0.02459580
##   0.0004037017  0.03201735  0.2180551  0.02459308
##   0.0004229243  0.03201211  0.2183732  0.02459115
##   0.0004430621  0.03200808  0.2186438  0.02458977
##   0.0004641589  0.03200456  0.2189003  0.02458951
##   0.0004862602  0.03200155  0.2191402  0.02459007
##   0.0005094138  0.03199948  0.2193436  0.02459184
##   0.0005336699  0.03199854  0.2195013  0.02459490
##   0.0005590810  0.03199832  0.2196341  0.02459877
##   0.0005857021  0.03199938  0.2197149  0.02460353
##   0.0006135907  0.03200055  0.2198031  0.02460804
##   0.0006428073  0.03200240  0.2198708  0.02461293
##   0.0006734151  0.03200492  0.2199173  0.02461764
##   0.0007054802  0.03200874  0.2199122  0.02462310
##   0.0007390722  0.03201346  0.2198770  0.02462914
##   0.0007742637  0.03201947  0.2197889  0.02463604
##   0.0008111308  0.03202592  0.2197064  0.02464268
##   0.0008497534  0.03203364  0.2195740  0.02465057
##   0.0008902151  0.03204343  0.2193511  0.02465944
##   0.0009326033  0.03205516  0.2190433  0.02466961
##   0.0009770100  0.03206837  0.2186811  0.02468145
##   0.0010235310  0.03208380  0.2182201  0.02469528
##   0.0010722672  0.03210033  0.2177294  0.02471028
##   0.0011233240  0.03211846  0.2171761  0.02472735
##   0.0011768120  0.03213470  0.2167630  0.02474330
##   0.0012328467  0.03215104  0.2163840  0.02475901
##   0.0012915497  0.03216667  0.2160860  0.02477452
##   0.0013530478  0.03218310  0.2157880  0.02479059
##   0.0014174742  0.03220140  0.2154357  0.02480948
##   0.0014849683  0.03222194  0.2150086  0.02483081
##   0.0015556761  0.03224445  0.2145226  0.02485464
##   0.0016297508  0.03226926  0.2139634  0.02488105
##   0.0017073526  0.03229652  0.2133225  0.02490917
##   0.0017886495  0.03232636  0.2125956  0.02493918
##   0.0018738174  0.03235829  0.2118186  0.02497095
##   0.0019630407  0.03239219  0.2110041  0.02500403
##   0.0020565123  0.03242568  0.2103095  0.02503698
##   0.0021544347  0.03246075  0.2096169  0.02507189
##   0.0022570197  0.03249606  0.2090355  0.02510712
##   0.0023644894  0.03253415  0.2084098  0.02514434
##   0.0024770764  0.03257588  0.2076910  0.02518442
##   0.0025950242  0.03262162  0.2068602  0.02522766
##   0.0027185882  0.03267174  0.2058957  0.02527416
##   0.0028480359  0.03272666  0.2047718  0.02532376
##   0.0029836472  0.03278684  0.2034570  0.02537698
##   0.0031257158  0.03285276  0.2019130  0.02543448
##   0.0032745492  0.03292496  0.2000921  0.02549745
##   0.0034304693  0.03300402  0.1979353  0.02556541
##   0.0035938137  0.03309058  0.1953695  0.02563875
##   0.0037649358  0.03318528  0.1923058  0.02571821
##   0.0039442061  0.03328586  0.1889009  0.02580270
##   0.0041320124  0.03339347  0.1850321  0.02589114
##   0.0043287613  0.03349602  0.1818112  0.02597489
##   0.0045348785  0.03360379  0.1783076  0.02606287
##   0.0047508102  0.03372085  0.1740859  0.02615709
##   0.0049770236  0.03384857  0.1688900  0.02625893
##   0.0052140083  0.03398754  0.1624760  0.02636955
##   0.0054622772  0.03413695  0.1547476  0.02648772
##   0.0057223677  0.03428323  0.1470087  0.02660214
##   0.0059948425  0.03443218  0.1385423  0.02671791
##   0.0062802914  0.03456278  0.1319602  0.02681402
##   0.0065793322  0.03469603  0.1247634  0.02690998
##   0.0068926121  0.03482128  0.1184698  0.02699800
##   0.0072208090  0.03495233  0.1109458  0.02709032
##   0.0075646333  0.03505674  0.1068584  0.02716450
##   0.0079248290  0.03515278  0.1042371  0.02723276
##   0.0083021757  0.03524229  0.1038025  0.02729509
##   0.0086974900  0.03533734  0.1038025  0.02736183
##   0.0091116276  0.03544136  0.1038025  0.02743474
##   0.0095454846  0.03555519  0.1038025  0.02751488
##   0.0100000000  0.03567970  0.1038025  0.02760324
## 
## Tuning parameter 'alpha' was held constant at a value of 1
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were alpha = 1 and lambda = 0.000559081.

##    alpha      lambda
## 38     1 0.000559081
##     alpha       lambda       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.0001000000 0.03228576 0.2064380 0.02476806 0.0008712829 0.03676116 0.0005768569
## 2       1 0.0001047616 0.03227771 0.2067390 0.02476232 0.0008708951 0.03685607 0.0005758343
## 3       1 0.0001097499 0.03226954 0.2070462 0.02475657 0.0008705209 0.03695773 0.0005748383
## 4       1 0.0001149757 0.03226119 0.2073622 0.02475076 0.0008702297 0.03706628 0.0005741082
## 5       1 0.0001204504 0.03225267 0.2076868 0.02474504 0.0008699447 0.03718157 0.0005734114
## 6       1 0.0001261857 0.03224387 0.2080246 0.02473914 0.0008695922 0.03730062 0.0005729025
## 7       1 0.0001321941 0.03223491 0.2083699 0.02473328 0.0008692183 0.03742283 0.0005725269
## 8       1 0.0001384886 0.03222580 0.2087234 0.02472756 0.0008687065 0.03754417 0.0005721548
## 9       1 0.0001450829 0.03221656 0.2090844 0.02472199 0.0008681960 0.03767055 0.0005720432
## 10      1 0.0001519911 0.03220742 0.2094424 0.02471646 0.0008679047 0.03779970 0.0005717913
## 11      1 0.0001592283 0.03219816 0.2098075 0.02471083 0.0008676298 0.03793179 0.0005714045
## 12      1 0.0001668101 0.03218869 0.2101859 0.02470498 0.0008673946 0.03807156 0.0005707816
## 13      1 0.0001747528 0.03217910 0.2105728 0.02469923 0.0008670571 0.03821218 0.0005698664
## 14      1 0.0001830738 0.03216926 0.2109742 0.02469306 0.0008666479 0.03835244 0.0005688149
## 15      1 0.0001917910 0.03215933 0.2113839 0.02468686 0.0008662711 0.03849779 0.0005677215
## 16      1 0.0002009233 0.03214933 0.2118013 0.02468058 0.0008661909 0.03864507 0.0005663444
## 17      1 0.0002104904 0.03213924 0.2122268 0.02467435 0.0008660061 0.03879461 0.0005648880
## 18      1 0.0002205131 0.03212904 0.2126618 0.02466789 0.0008656690 0.03893572 0.0005630387
## 19      1 0.0002310130 0.03211883 0.2131024 0.02466143 0.0008652569 0.03907659 0.0005609128
## 20      1 0.0002420128 0.03210841 0.2135598 0.02465457 0.0008643713 0.03922826 0.0005584429
## 21      1 0.0002535364 0.03209829 0.2140101 0.02464786 0.0008633423 0.03938504 0.0005559140
## 22      1 0.0002656088 0.03208856 0.2144518 0.02464084 0.0008620351 0.03956064 0.0005535882
## 23      1 0.0002782559 0.03207945 0.2148724 0.02463401 0.0008604190 0.03974433 0.0005512395
## 24      1 0.0002915053 0.03207058 0.2152909 0.02462730 0.0008585401 0.03991542 0.0005483817
## 25      1 0.0003053856 0.03206206 0.2157013 0.02462064 0.0008564390 0.04008625 0.0005451657
## 26      1 0.0003199267 0.03205339 0.2161297 0.02461439 0.0008540551 0.04025166 0.0005415283
## 27      1 0.0003351603 0.03204509 0.2165493 0.02460858 0.0008513204 0.04040255 0.0005378772
## 28      1 0.0003511192 0.03203737 0.2169497 0.02460353 0.0008484707 0.04055637 0.0005340412
## 29      1 0.0003678380 0.03203043 0.2173232 0.02459932 0.0008452682 0.04070167 0.0005296175
## 30      1 0.0003853529 0.03202361 0.2176976 0.02459580 0.0008414949 0.04080626 0.0005247790
## 31      1 0.0004037017 0.03201735 0.2180551 0.02459308 0.0008374052 0.04089338 0.0005195409
## 32      1 0.0004229243 0.03201211 0.2183732 0.02459115 0.0008327949 0.04097045 0.0005135050
## 33      1 0.0004430621 0.03200808 0.2186438 0.02458977 0.0008278291 0.04103836 0.0005072873
## 34      1 0.0004641589 0.03200456 0.2189003 0.02458951 0.0008227433 0.04107595 0.0005007692
## 35      1 0.0004862602 0.03200155 0.2191402 0.02459007 0.0008177335 0.04108651 0.0004934100
## 36      1 0.0005094138 0.03199948 0.2193436 0.02459184 0.0008129325 0.04105391 0.0004866174
## 37      1 0.0005336699 0.03199854 0.2195013 0.02459490 0.0008077224 0.04097500 0.0004794737
## 38      1 0.0005590810 0.03199832 0.2196341 0.02459877 0.0008023265 0.04087601 0.0004724684
## 39      1 0.0005857021 0.03199938 0.2197149 0.02460353 0.0007967162 0.04075587 0.0004656081
## 40      1 0.0006135907 0.03200055 0.2198031 0.02460804 0.0007912155 0.04062684 0.0004594653
## 41      1 0.0006428073 0.03200240 0.2198708 0.02461293 0.0007860947 0.04049816 0.0004532403
## 42      1 0.0006734151 0.03200492 0.2199173 0.02461764 0.0007816584 0.04039912 0.0004477080
## 43      1 0.0007054802 0.03200874 0.2199122 0.02462310 0.0007776225 0.04029783 0.0004429947
## 44      1 0.0007390722 0.03201346 0.2198770 0.02462914 0.0007747006 0.04019551 0.0004392131
## 45      1 0.0007742637 0.03201947 0.2197889 0.02463604 0.0007716284 0.04007150 0.0004358775
## 46      1 0.0008111308 0.03202592 0.2197064 0.02464268 0.0007685012 0.03997913 0.0004335174
## 47      1 0.0008497534 0.03203364 0.2195740 0.02465057 0.0007652464 0.03986630 0.0004313486
## 48      1 0.0008902151 0.03204343 0.2193511 0.02465944 0.0007610420 0.03967315 0.0004283191
## 49      1 0.0009326033 0.03205516 0.2190433 0.02466961 0.0007569148 0.03945970 0.0004252546
## 50      1 0.0009770100 0.03206837 0.2186811 0.02468145 0.0007528138 0.03929506 0.0004216231
## 51      1 0.0010235310 0.03208380 0.2182201 0.02469528 0.0007485090 0.03911976 0.0004179498
## 52      1 0.0010722672 0.03210033 0.2177294 0.02471028 0.0007436144 0.03895638 0.0004149421
## 53      1 0.0011233240 0.03211846 0.2171761 0.02472735 0.0007384787 0.03881892 0.0004121102
## 54      1 0.0011768120 0.03213470 0.2167630 0.02474330 0.0007320138 0.03868175 0.0004089346
## 55      1 0.0012328467 0.03215104 0.2163840 0.02475901 0.0007258335 0.03853015 0.0004060597
## 56      1 0.0012915497 0.03216667 0.2160860 0.02477452 0.0007198690 0.03844135 0.0004040509
## 57      1 0.0013530478 0.03218310 0.2157880 0.02479059 0.0007145899 0.03832341 0.0004022662
## 58      1 0.0014174742 0.03220140 0.2154357 0.02480948 0.0007105204 0.03829259 0.0004006920
## 59      1 0.0014849683 0.03222194 0.2150086 0.02483081 0.0007068315 0.03826990 0.0003998271
## 60      1 0.0015556761 0.03224445 0.2145226 0.02485464 0.0007039670 0.03825033 0.0003987774
## 61      1 0.0016297508 0.03226926 0.2139634 0.02488105 0.0007011324 0.03822533 0.0003975780
## 62      1 0.0017073526 0.03229652 0.2133225 0.02490917 0.0006981089 0.03817832 0.0003958059
## 63      1 0.0017886495 0.03232636 0.2125956 0.02493918 0.0006949747 0.03811556 0.0003944498
## 64      1 0.0018738174 0.03235829 0.2118186 0.02497095 0.0006928792 0.03800505 0.0003932537
## 65      1 0.0019630407 0.03239219 0.2110041 0.02500403 0.0006913812 0.03789463 0.0003912399
## 66      1 0.0020565123 0.03242568 0.2103095 0.02503698 0.0006905742 0.03782107 0.0003893355
## 67      1 0.0021544347 0.03246075 0.2096169 0.02507189 0.0006906585 0.03767787 0.0003879456
## 68      1 0.0022570197 0.03249606 0.2090355 0.02510712 0.0006899262 0.03757777 0.0003860563
## 69      1 0.0023644894 0.03253415 0.2084098 0.02514434 0.0006895135 0.03745045 0.0003843998
## 70      1 0.0024770764 0.03257588 0.2076910 0.02518442 0.0006893481 0.03730462 0.0003830127
## 71      1 0.0025950242 0.03262162 0.2068602 0.02522766 0.0006894645 0.03713652 0.0003820396
## 72      1 0.0027185882 0.03267174 0.2058957 0.02527416 0.0006899028 0.03694181 0.0003820368
## 73      1 0.0028480359 0.03272666 0.2047718 0.02532376 0.0006907088 0.03671547 0.0003825903
## 74      1 0.0029836472 0.03278684 0.2034570 0.02537698 0.0006919327 0.03645137 0.0003838695
## 75      1 0.0031257158 0.03285276 0.2019130 0.02543448 0.0006936289 0.03614201 0.0003855535
## 76      1 0.0032745492 0.03292496 0.2000921 0.02549745 0.0006958566 0.03577825 0.0003875407
## 77      1 0.0034304693 0.03300402 0.1979353 0.02556541 0.0006986792 0.03534887 0.0003908716
## 78      1 0.0035938137 0.03309058 0.1953695 0.02563875 0.0007021645 0.03484016 0.0003950751
## 79      1 0.0037649358 0.03318528 0.1923058 0.02571821 0.0007063783 0.03423995 0.0004012024
## 80      1 0.0039442061 0.03328586 0.1889009 0.02580270 0.0007103680 0.03364602 0.0004077014
## 81      1 0.0041320124 0.03339347 0.1850321 0.02589114 0.0007153055 0.03317041 0.0004140269
## 82      1 0.0043287613 0.03349602 0.1818112 0.02597489 0.0007152004 0.03295653 0.0004154521
## 83      1 0.0045348785 0.03360379 0.1783076 0.02606287 0.0007170796 0.03240156 0.0004179803
## 84      1 0.0047508102 0.03372085 0.1740859 0.02615709 0.0007204269 0.03173004 0.0004213301
## 85      1 0.0049770236 0.03384857 0.1688900 0.02625893 0.0007245589 0.03089707 0.0004271334
## 86      1 0.0052140083 0.03398754 0.1624760 0.02636955 0.0007300151 0.02978543 0.0004353377
## 87      1 0.0054622772 0.03413695 0.1547476 0.02648772 0.0007375872 0.02857546 0.0004451220
## 88      1 0.0057223677 0.03428323 0.1470087 0.02660214 0.0007442228 0.02716279 0.0004566132
## 89      1 0.0059948425 0.03443218 0.1385423 0.02671791 0.0007550701 0.02533534 0.0004708915
## 90      1 0.0062802914 0.03456278 0.1319602 0.02681402 0.0007611921 0.02463343 0.0004809028
## 91      1 0.0065793322 0.03469603 0.1247634 0.02690998 0.0007692642 0.02422339 0.0004918811
## 92      1 0.0068926121 0.03482128 0.1184698 0.02699800 0.0007718331 0.02491174 0.0004953887
## 93      1 0.0072208090 0.03495233 0.1109458 0.02709032 0.0007779862 0.02530549 0.0004996236
## 94      1 0.0075646333 0.03505674 0.1068584 0.02716450 0.0007804892 0.02476156 0.0005034510
## 95      1 0.0079248290 0.03515278 0.1042371 0.02723276 0.0007920608 0.02329182 0.0005120963
## 96      1 0.0083021757 0.03524229 0.1038025 0.02729509 0.0007990456 0.02345731 0.0005188893
## 97      1 0.0086974900 0.03533734 0.1038025 0.02736183 0.0008080099 0.02345731 0.0005273361
## 98      1 0.0091116276 0.03544136 0.1038025 0.02743474 0.0008175321 0.02345731 0.0005364083
## 99      1 0.0095454846 0.03555519 0.1038025 0.02751488 0.0008276366 0.02345731 0.0005445086
## 100     1 0.0100000000 0.03567970 0.1038025 0.02760324 0.0008383470 0.02345731 0.0005521625

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##                model.coef
## (Intercept)  1.999620e+00
## x4          -3.192729e-05
## x5           1.298986e-04
## x7           1.026337e-02
## x8           2.468266e-04
## x9           2.854969e-03
## x10          7.169840e-04
## x11          6.360430e+04
## x14         -9.023026e-06
## x16          4.469093e-04
## x17          1.098423e-03
## x19          1.193228e-05
## x21          6.167787e-05
## x22         -7.402228e-06
## stat3        4.051790e-05
## stat4       -6.713828e-05
## stat13      -3.091034e-04
## stat14      -3.997486e-04
## stat15      -1.691767e-05
## stat22      -2.998355e-04
## stat23       4.173617e-04
## stat24      -3.792153e-04
## stat25      -7.594691e-05
## stat26      -5.072840e-06
## stat30       4.589487e-05
## stat35      -4.278460e-05
## stat37      -9.091416e-05
## stat38       8.747548e-05
## stat39      -4.550145e-05
## stat41      -1.986573e-04
## stat43      -4.992256e-05
## stat45      -2.277058e-06
## stat50       8.465589e-05
## stat51       1.792312e-04
## stat54      -8.773038e-05
## stat55       2.978606e-05
## stat59       8.998934e-05
## stat60       8.193966e-05
## stat63       1.061022e-04
## stat65      -9.633573e-06
## stat75      -8.206138e-07
## stat87      -1.393615e-04
## stat91      -8.354389e-05
## stat92      -2.171178e-04
## stat94      -3.246685e-05
## stat95      -1.720130e-04
## stat98       3.231148e-03
## stat99       1.796786e-05
## stat100      2.543515e-04
## stat104     -5.513857e-05
## stat106     -1.797958e-04
## stat110     -2.978722e-03
## stat112     -8.945143e-06
## stat113     -1.452536e-05
## stat115      6.441635e-06
## stat130      5.114863e-05
## stat144      2.686242e-04
## stat146     -1.326285e-04
## stat147     -6.098621e-05
## stat149     -2.858630e-04
## stat152     -1.508949e-05
## stat156      1.688586e-04
## stat170     -1.565258e-04
## stat172      8.751617e-05
## stat173     -1.979953e-05
## stat175     -1.686258e-04
## stat185     -7.067548e-05
## stat187     -1.034201e-04
## stat191     -1.990295e-05
## stat199      4.584725e-05
## stat204     -1.391253e-04
## stat207      1.668615e-05
## stat210     -1.210908e-05
## stat214     -2.243818e-05
## stat217      3.725489e-05
## x18.sqrt     2.409284e-02

Test

if (algo.LASSO.caret == TRUE){
  test.model(model.LASSO.caret, data.test
             ,method = 'glmnet',subopt = "LASSO"
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}

## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.041   2.085   2.097   2.097   2.108   2.148 
## [1] "glmnet LASSO Test MSE: 0.000942657388743223"

LARS with CV

Train

if (algo.LARS.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "lars"
                                   ,subopt = 'NULL'
                                   ,feature.names = feature.names)
  model.LARS.caret = returned$model
}

## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, : There were missing values in resampled
## performance measures.

## Aggregating results
## Selecting tuning parameters
## Fitting fraction = 0.384 on full training set
## Least Angle Regression 
## 
## 5584 samples
##  240 predictor
## 
## Pre-processing: centered (240), scaled (240) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   fraction    RMSE        Rsquared   MAE       
##   0.00000000  0.03615283        NaN  0.02793755
##   0.01010101  0.03573595  0.1038025  0.02764360
##   0.02020202  0.03536764  0.1038025  0.02738318
##   0.03030303  0.03505460  0.1052881  0.02716424
##   0.04040404  0.03477639  0.1206858  0.02696601
##   0.05050505  0.03452627  0.1332179  0.02678628
##   0.06060606  0.03429091  0.1462363  0.02660741
##   0.07070707  0.03407398  0.1582559  0.02643766
##   0.08080808  0.03386395  0.1684154  0.02627123
##   0.09090909  0.03366691  0.1762344  0.02611440
##   0.10101010  0.03348307  0.1822212  0.02596501
##   0.11111111  0.03331393  0.1878454  0.02582624
##   0.12121212  0.03315406  0.1934511  0.02569254
##   0.13131313  0.03300481  0.1979850  0.02556713
##   0.14141414  0.03286642  0.2016245  0.02544747
##   0.15151515  0.03273902  0.2045335  0.02533540
##   0.16161616  0.03262273  0.2068473  0.02522986
##   0.17171717  0.03251769  0.2086771  0.02512989
##   0.18181818  0.03242686  0.2102334  0.02503952
##   0.19191919  0.03234820  0.2120182  0.02496156
##   0.20202020  0.03227757  0.2136557  0.02489123
##   0.21212121  0.03221744  0.2149915  0.02482799
##   0.22222222  0.03216837  0.2160892  0.02477678
##   0.23232323  0.03213206  0.2169287  0.02474073
##   0.24242424  0.03210360  0.2176151  0.02471342
##   0.25252525  0.03207805  0.2183397  0.02468988
##   0.26262626  0.03205588  0.2189945  0.02467061
##   0.27272727  0.03203816  0.2195049  0.02465462
##   0.28282828  0.03202650  0.2197453  0.02464434
##   0.29292929  0.03201865  0.2198456  0.02463728
##   0.30303030  0.03201193  0.2199464  0.02463065
##   0.31313131  0.03200638  0.2200249  0.02462417
##   0.32323232  0.03200233  0.2200526  0.02461863
##   0.33333333  0.03199927  0.2200467  0.02461418
##   0.34343434  0.03199698  0.2200219  0.02460976
##   0.35353535  0.03199531  0.2199737  0.02460510
##   0.36363636  0.03199431  0.2199048  0.02460115
##   0.37373737  0.03199376  0.2198259  0.02459752
##   0.38383838  0.03199364  0.2197357  0.02459424
##   0.39393939  0.03199404  0.2196243  0.02459155
##   0.40404040  0.03199520  0.2194817  0.02458960
##   0.41414141  0.03199666  0.2193325  0.02458791
##   0.42424242  0.03199854  0.2191689  0.02458667
##   0.43434343  0.03200121  0.2189702  0.02458636
##   0.44444444  0.03200412  0.2187664  0.02458632
##   0.45454545  0.03200662  0.2185890  0.02458684
##   0.46464646  0.03200973  0.2183854  0.02458806
##   0.47474747  0.03201334  0.2181640  0.02458983
##   0.48484848  0.03201747  0.2179225  0.02459174
##   0.49494949  0.03202206  0.2176636  0.02459396
##   0.50505051  0.03202699  0.2173928  0.02459667
##   0.51515152  0.03203189  0.2171270  0.02459951
##   0.52525253  0.03203707  0.2168535  0.02460280
##   0.53535354  0.03204265  0.2165663  0.02460652
##   0.54545455  0.03204834  0.2162800  0.02461058
##   0.55555556  0.03205454  0.2159744  0.02461510
##   0.56565657  0.03206103  0.2156616  0.02461988
##   0.57575758  0.03206734  0.2153615  0.02462474
##   0.58585859  0.03207371  0.2150634  0.02462974
##   0.59595960  0.03208031  0.2147571  0.02463489
##   0.60606061  0.03208718  0.2144434  0.02464022
##   0.61616162  0.03209412  0.2141318  0.02464523
##   0.62626263  0.03210154  0.2138030  0.02465031
##   0.63636364  0.03210961  0.2134474  0.02465578
##   0.64646465  0.03211780  0.2130919  0.02466121
##   0.65656566  0.03212607  0.2127368  0.02466661
##   0.66666667  0.03213448  0.2123796  0.02467199
##   0.67676768  0.03214290  0.2120252  0.02467722
##   0.68686869  0.03215138  0.2116711  0.02468248
##   0.69696970  0.03215990  0.2113191  0.02468776
##   0.70707071  0.03216853  0.2109663  0.02469306
##   0.71717172  0.03217748  0.2106033  0.02469854
##   0.72727273  0.03218650  0.2102415  0.02470396
##   0.73737374  0.03219549  0.2098855  0.02470944
##   0.74747475  0.03220458  0.2095294  0.02471504
##   0.75757576  0.03221380  0.2091706  0.02472046
##   0.76767677  0.03222364  0.2087873  0.02472650
##   0.77777778  0.03223350  0.2084061  0.02473281
##   0.78787879  0.03224373  0.2080129  0.02473949
##   0.79797980  0.03225424  0.2076112  0.02474647
##   0.80808081  0.03226462  0.2072182  0.02475338
##   0.81818182  0.03227524  0.2068195  0.02476080
##   0.82828283  0.03228628  0.2064073  0.02476852
##   0.83838384  0.03229763  0.2059868  0.02477634
##   0.84848485  0.03230932  0.2055568  0.02478439
##   0.85858586  0.03232126  0.2051197  0.02479275
##   0.86868687  0.03233347  0.2046751  0.02480146
##   0.87878788  0.03234573  0.2042320  0.02481032
##   0.88888889  0.03235800  0.2037928  0.02481936
##   0.89898990  0.03237031  0.2033572  0.02482865
##   0.90909091  0.03238276  0.2029196  0.02483809
##   0.91919192  0.03239530  0.2024832  0.02484752
##   0.92929293  0.03240791  0.2020484  0.02485715
##   0.93939394  0.03242076  0.2016072  0.02486700
##   0.94949495  0.03243379  0.2011640  0.02487706
##   0.95959596  0.03244720  0.2007096  0.02488780
##   0.96969697  0.03246083  0.2002492  0.02489874
##   0.97979798  0.03247458  0.1997882  0.02490974
##   0.98989899  0.03248853  0.1993232  0.02492083
##   1.00000000  0.03250253  0.1988622  0.02493193
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was fraction = 0.3838384.

##     fraction
## 39 0.3838384

## Warning: Removed 1 rows containing missing values (geom_point).

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##            x4            x5            x7            x8            x9           x10           x11           x14 
## -1.531793e-03  1.977061e-04  6.911796e-03  7.247751e-04  3.706721e-03  1.006858e-03  3.800232e-04 -1.011551e-05 
##           x16           x17           x19           x21           x22         stat3         stat4        stat13 
##  9.058036e-04  1.460923e-03  3.862172e-05  6.361488e-04 -4.712600e-06  8.007205e-05 -1.246502e-04 -5.447453e-04 
##        stat14        stat15        stat22        stat23        stat24        stat25        stat26        stat30 
## -7.104915e-04 -3.829481e-05 -5.273807e-04  7.329366e-04 -6.632203e-04 -1.422530e-04 -2.263488e-06  8.680800e-05 
##        stat35        stat37        stat38        stat39        stat41        stat43        stat50        stat51 
## -8.469060e-05 -1.656486e-04  1.616390e-04 -8.751430e-05 -3.525696e-04 -9.516236e-05  1.576032e-04  3.178300e-04 
##        stat54        stat55        stat59        stat60        stat63        stat65        stat87        stat91 
## -1.606147e-04  6.081155e-05  1.619828e-04  1.522761e-04  1.911492e-04 -1.513368e-05 -2.505654e-04 -1.540178e-04 
##        stat92        stat94        stat95        stat98        stat99       stat100       stat104       stat106 
## -3.809821e-04 -6.608045e-05 -3.077679e-04  5.701316e-03  3.993515e-05  4.540061e-04 -1.050582e-04 -3.217559e-04 
##       stat110       stat112       stat113       stat115       stat130       stat144       stat146       stat147 
## -5.161655e-03 -1.570879e-05 -3.178462e-05  8.748157e-06  9.613664e-05  4.754582e-04 -2.384243e-04 -1.153419e-04 
##       stat149       stat152       stat156       stat170       stat172       stat173       stat175       stat185 
## -5.013289e-04 -3.340497e-05  3.000084e-04 -2.786896e-04  1.615455e-04 -4.349482e-05 -3.027524e-04 -1.340565e-04 
##       stat187       stat191       stat199       stat204       stat207       stat210       stat214       stat217 
## -1.897762e-04 -4.377823e-05  9.031433e-05 -2.532868e-04  3.643533e-05 -2.479095e-05 -4.731099e-05  7.200011e-05 
##      x18.sqrt 
##  1.093130e-02

Test

if (algo.LARS.caret == TRUE){
  test.model(model.LARS.caret, data.test
             ,method = 'lars',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}

## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.040   2.085   2.097   2.097   2.108   2.148 
## [1] "lars  Test MSE: 0.000942692945958325"

Session Info

sessionInfo()

## R version 3.5.2 (2018-12-20)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 17763)
## 
## Matrix products: default
## 
## locale:
## [1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
## [4] LC_NUMERIC=C                           LC_TIME=English_United States.1252    
## 
## attached base packages:
## [1] parallel  stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] knitr_1.21                 htmltools_0.3.6            reshape2_1.4.3             lars_1.2                  
##  [5] doParallel_1.0.14          iterators_1.0.10           caret_6.0-81               leaps_3.0                 
##  [9] ggforce_0.1.3              rlist_0.4.6.1              car_3.0-2                  carData_3.0-2             
## [13] bestNormalize_1.3.0        scales_1.0.0               onewaytests_2.0            caTools_1.17.1.1          
## [17] mosaic_1.5.0               mosaicData_0.17.0          ggformula_0.9.1            ggstance_0.3.1            
## [21] lattice_0.20-38            DT_0.5                     ggiraphExtra_0.2.9         ggiraph_0.6.0             
## [25] investr_1.4.0              glmnet_2.0-16              foreach_1.4.4              Matrix_1.2-15             
## [29] MASS_7.3-51.1              PerformanceAnalytics_1.5.2 xts_0.11-2                 zoo_1.8-4                 
## [33] forcats_0.3.0              stringr_1.4.0              dplyr_0.8.0.1              purrr_0.3.0               
## [37] readr_1.3.1                tidyr_0.8.2                tibble_2.0.1               ggplot2_3.1.0             
## [41] tidyverse_1.2.1            usdm_1.1-18                raster_2.8-19              sp_1.3-1                  
## [45] pacman_0.5.0              
## 
## loaded via a namespace (and not attached):
##  [1] readxl_1.3.0       backports_1.1.3    plyr_1.8.4         lazyeval_0.2.1     splines_3.5.2      mycor_0.1.1       
##  [7] crosstalk_1.0.0    leaflet_2.0.2      digest_0.6.18      magrittr_1.5       mosaicCore_0.6.0   openxlsx_4.1.0    
## [13] recipes_0.1.4      modelr_0.1.3       gower_0.1.2        colorspace_1.4-0   rvest_0.3.2        ggrepel_0.8.0     
## [19] haven_2.0.0        xfun_0.4           crayon_1.3.4       jsonlite_1.6       survival_2.43-3    glue_1.3.0        
## [25] registry_0.5       gtable_0.2.0       ppcor_1.1          ipred_0.9-8        sjmisc_2.7.7       abind_1.4-5       
## [31] rngtools_1.3.1     bibtex_0.4.2       Rcpp_1.0.0         xtable_1.8-3       units_0.6-2        foreign_0.8-71    
## [37] stats4_3.5.2       lava_1.6.5         prodlim_2018.04.18 prediction_0.3.6.2 htmlwidgets_1.3    httr_1.4.0        
## [43] RColorBrewer_1.1-2 pkgconfig_2.0.2    farver_1.1.0       nnet_7.3-12        labeling_0.3       tidyselect_0.2.5  
## [49] rlang_0.3.1        later_0.8.0        munsell_0.5.0      cellranger_1.1.0   tools_3.5.2        cli_1.0.1         
## [55] generics_0.0.2     moments_0.14       sjlabelled_1.0.16  broom_0.5.1        evaluate_0.13      ggdendro_0.1-20   
## [61] yaml_2.2.0         ModelMetrics_1.2.2 zip_1.0.0          nlme_3.1-137       doRNG_1.7.1        mime_0.6          
## [67] xml2_1.2.0         compiler_3.5.2     rstudioapi_0.9.0   curl_3.3           tweenr_1.0.1       stringi_1.3.1     
## [73] highr_0.7          gdtools_0.1.7      stringdist_0.9.5.1 pillar_1.3.1       data.table_1.12.0  bitops_1.0-6      
## [79] httpuv_1.4.5.1     R6_2.4.0           promises_1.0.1     gridExtra_2.3      rio_0.5.16         codetools_0.2-15  
## [85] assertthat_0.2.0   pkgmaker_0.27      withr_2.1.2        nortest_1.0-4      mgcv_1.8-26        hms_0.4.2         
## [91] quadprog_1.5-5     grid_3.5.2         rpart_4.1-13       timeDate_3043.102  class_7.3-14       rmarkdown_1.11    
## [97] snakecase_0.9.2    shiny_1.2.0        lubridate_1.7.4